diff --git a/src/app/filter/filter.component.ts b/src/app/filter/filter.component.ts
index 1e78645a..a1d21756 100644
--- a/src/app/filter/filter.component.ts
+++ b/src/app/filter/filter.component.ts
@@ -4,7 +4,7 @@ import { skip } from 'rxjs/operators';
 import { Options, OptionType } from '@app/shared/model/options.model';
 import { Filter } from '@app/shared/model/filter.model';
 import { Settings } from '@app/shared/model/settings.model';
-import { Note } from '@app/shared/model/notes.model';
+import { Kep } from '@app/shared/model/notes.model';
 import { getAllNotesSelector } from '@app/notes/notes.reducer';
 import { State } from '@app/app.reducer';
 import { UpdateFilter } from './filter.actions';
@@ -20,7 +20,7 @@ export class FilterComponent implements OnInit {
   options: Options = new Options();
   filter: Filter = new Filter();
   settings: Settings = new Settings();
-  notes: Note[] = [];
+  keps: Kep[] = [];
 
   /**
    * FilterComponent's constructor
@@ -31,9 +31,9 @@ export class FilterComponent implements OnInit {
    * Runs after component initialization
    */
   ngOnInit() {
-    this.store.pipe(select(getAllNotesSelector)).subscribe(notes => {
-      if (notes) {
-        this.notes = notes;
+    this.store.pipe(select(getAllNotesSelector)).subscribe(keps => {
+      if (keps) {
+        this.keps = keps;
         this.updateOptions();
       }
     });
@@ -54,34 +54,19 @@ export class FilterComponent implements OnInit {
   }
 
   /**
-   * Update the options from the provided notes
+   * Update the options from the provided keps
    */
   updateOptions(): void {
     // Reset the options
     this.options = new Options();
 
     // Populate the new options
-    for (const note of Object.values(this.notes)) {
-      if (OptionType.areas in note) {
-        this.options.add(OptionType.areas, note.areas);
+    for (const kep of Object.values(this.keps)) {
+      if (OptionType.owningSigs in kep) {
+        this.options.add(OptionType.owningSigs, [kep.owningSig]);
       }
-      if (OptionType.kinds in note) {
-        this.options.add(OptionType.kinds, note.kinds);
-      }
-      if (OptionType.sigs in note) {
-        this.options.add(OptionType.sigs, note.sigs);
-      }
-      if (OptionType.documentation in note) {
-        this.options.add(
-          OptionType.documentation,
-          note.documentation.map(x => x.type.toString()),
-        );
-      }
-      if (
-        !this.options.get(OptionType.releaseVersions).has(note.release_version) &&
-        (this.settings.displayPreReleases || !this.filter.isPreRelease(note.release_version))
-      ) {
-        this.options.get(OptionType.releaseVersions).add(note.release_version);
+      if (OptionType.participatingSigs in kep) {
+        this.options.add(OptionType.participatingSigs, kep.participatingSigs);
       }
     }
 
diff --git a/src/app/main/main.component.html b/src/app/main/main.component.html
index cd9d2538..3f642294 100644
--- a/src/app/main/main.component.html
+++ b/src/app/main/main.component.html
@@ -1,7 +1,7 @@
 <nav class="navbar fixed-top navbar-dark bg-dark navbar-expand-lg shadow">
   <a class="navbar-brand" href="/">
     <img src="assets/img/logo.svg" alt="kubernetes" class="d-inline-block align-top" />
-    release notes
+    Enhancement Proposals
   </a>
   <button
     class="navbar-toggler"
diff --git a/src/app/notes/notes.actions.ts b/src/app/notes/notes.actions.ts
index 3ebb8587..78af1937 100644
--- a/src/app/notes/notes.actions.ts
+++ b/src/app/notes/notes.actions.ts
@@ -1,5 +1,5 @@
 import { Action } from '@ngrx/store';
-import { Note } from '@app/shared/model/notes.model';
+import { Kep } from '@app/shared/model/notes.model';
 import { Filter } from '@app/shared/model/filter.model';
 
 export enum ActionTypes {
@@ -19,12 +19,12 @@ export class Failed implements Action {
 
 export class DoFilter implements Action {
   readonly type = ActionTypes.DoFilter;
-  constructor(public notes: Note[], public filter: Filter) {}
+  constructor(public notes: Kep[], public filter: Filter) {}
 }
 
 export class DoFilterSuccess implements Action {
   readonly type = ActionTypes.DoFilterSuccess;
-  constructor(public notes: Note[]) {}
+  constructor(public notes: Kep[]) {}
 }
 
 export class GetNotes implements Action {
@@ -34,7 +34,7 @@ export class GetNotes implements Action {
 
 export class GetNotesSuccess implements Action {
   readonly type = ActionTypes.GetNotesSuccess;
-  constructor(public payload: Note[]) {}
+  constructor(public payload: Kep[]) {}
 }
 
 export type NotesAction = Failed | DoFilter | DoFilterSuccess | GetNotes | GetNotesSuccess;
diff --git a/src/app/notes/notes.component.html b/src/app/notes/notes.component.html
index 955d2b75..47d97f5b 100644
--- a/src/app/notes/notes.component.html
+++ b/src/app/notes/notes.component.html
@@ -4,64 +4,39 @@
 <ng-container *ngFor="let note of filteredNotes | paginate: { itemsPerPage: 10, currentPage: p }">
   <div class="card">
     <div class="card-header">
-      <a target="_blank" href="{{ note.pr_url }}">#{{ note.pr_number }}</a> by
-      <a target="_blank" href="{{ note.author_url }}">{{ note.author }}</a>
+      <a target="_blank" href="#">#{{ note.title }}</a> by
+      <a target="_blank" href="#">{{ note.authors }} </a>
+      <span>(owned by {{ note.owningSig }})</span>
+      <br />
+      <span>{{ note.creationDate }} (Updated: {{ note.lastUpdated }})</span>
+
       <a
-        ngbTooltip="Additional Documentation"
+        ngbTooltip="Show"
         placement="right"
         tooltipClass="documentationTooltip"
-        *ngIf="note.documentation"
         class="text-secondary"
-        id="documentationButton-{{ note.pr_number }}"
+        id="documentationButton-{{ note.id }}"
         data-toggle="collapse"
-        href="#documentationContent-{{ note.pr_number }}"
+        href="#documentationContent-{{ note.id }}"
       >
         <fa-icon class="icon" [icon]="faBook" size="lg"></fa-icon>
       </a>
-      <span
-        (click)="toggleFilter(OptionType.releaseVersions, note.release_version)"
-        class="k8s-badge float-sm-right badge badge-dark"
-        >{{ note.release_version }}</span
-      >
 
+      <span class="k8s-badge float-sm-right badge badge-dark"
+        >Reviewed by: {{ note.reviewers }}</span
+      >
       <span
-        *ngFor="let sig of note.sigs"
-        (click)="toggleFilter(OptionType.sigs, sig)"
+        *ngFor="let sig of note.participatingSigs"
         class="k8s-badge float-sm-right badge badge-success"
         >{{ sig }}</span
       >
-      <span
-        *ngFor="let kind of note.kinds"
-        (click)="toggleFilter(OptionType.kinds, kind)"
-        class="k8s-badge float-sm-right badge badge-primary"
-        >{{ kind }}</span
-      >
-      <span
-        *ngFor="let area of note.areas"
-        (click)="toggleFilter(OptionType.areas, area)"
-        class="k8s-badge float-sm-right badge badge-secondary"
-        >{{ area }}</span
-      >
+      <span class="k8s-badge float-sm-right badge badge-success">Editor: {{ note.editor }}</span>
+      <span class="k8s-badge float-sm-right badge badge-primary">{{ note.status }}</span>
     </div>
     <div class="card-body">
-      <div [innerHTML]="note.markdown | markdown"></div>
-      <div
-        *ngIf="note.documentation"
-        class="collapse {{ collapseClass() }}"
-        id="documentationContent-{{ note.pr_number }}"
-      >
-        <h6>Additional Documentation</h6>
-        <ul>
-          <li *ngFor="let doc of note.documentation">
-            <a href="{{ doc.url }}" target="_blank">
-              <span *ngIf="doc.description; else elseBlock">
-                {{ saneKEPDescription(doc) }}
-              </span>
-              <ng-template #elseBlock>{{ doc.url }}</ng-template>
-              <span class="badge doc-badge {{ badgeClass(doc.type) }}">{{ doc.type }}</span>
-            </a>
-          </li>
-        </ul>
+      <div [innerHTML]="toc(note.markdown) | markdown"></div>
+      <div class="collapse {{ collapseClass() }}" id="documentationContent-{{ note.id }}">
+        <div [innerHTML]="stripToc(note.markdown) | markdown"></div>
       </div>
     </div>
   </div>
diff --git a/src/app/notes/notes.component.ts b/src/app/notes/notes.component.ts
index 4d1937c4..cf187dcc 100644
--- a/src/app/notes/notes.component.ts
+++ b/src/app/notes/notes.component.ts
@@ -1,6 +1,6 @@
 import { Component } from '@angular/core';
 import { Store, select } from '@ngrx/store';
-import { Note, Documentation } from '@app/shared/model/notes.model';
+import { Kep } from '@app/shared/model/notes.model';
 import { DoFilter, GetNotes } from './notes.actions';
 import { State } from '@app/app.reducer';
 import { getErrorSelector, getAllNotesSelector, getFilteredNotesSelector } from './notes.reducer';
@@ -18,13 +18,14 @@ import { getFilterSelector } from '@app/filter/filter.reducer';
 })
 export class NotesComponent {
   filter: Filter = new Filter();
-  allNotes: Note[] = [];
-  filteredNotes: Note[] = [];
+  allNotes: Kep[] = [];
+  filteredNotes: Kep[] = [];
   p = 1;
   faBook = faBook;
   errorMessage = '';
 
-  readonly kep = 'KEP';
+  readonly tocStart = '<!-- toc -->';
+  readonly tocEnd = '<!-- /toc -->';
 
   constructor(private store: Store<State>) {
     this.store.dispatch(new GetNotes());
@@ -74,50 +75,37 @@ export class NotesComponent {
   }
 
   /**
-   * Retrieve the badge css class for a given documentation string
+   * Retrieve the collapse css class based on the current filter
    *
    * @returns The resulting class as string
    */
-  public badgeClass(t: string): string {
-    if (t === this.kep) {
-      return 'badge-primary';
-    } else if (t === 'official') {
-      return 'badge-success';
-    }
-    return 'badge-secondary';
+  public collapseClass(): string {
+    /* if (!this.filter.optionIsEmpty(OptionType.owningSigs)) {
+      return 'show';
+    } */
+    return '';
   }
 
   /**
-   * Sanitize the documentation description
-   *
-   * @param doc The documentation to be processed
+   * Returns the table of contents from a markdown string
    *
-   * @returns The resulting description as string
+   * @returns The resulting string
    */
-  public saneKEPDescription(doc: Documentation): string {
-    const stripped = doc.description
-      .replace(/[\[\]]/g, '') // remove brackets
-      .replace(this.kep, '') // remove 'KEP'
-      .trim();
-
-    if (stripped === '') {
-      // write out KEP
-      return 'Kubernetes Enhancement Proposal';
-    }
-
-    // all other sort of descriptions
-    return stripped;
+  public toc(md: string): string {
+    const toc = md.substring(
+      md.lastIndexOf(this.tocStart) + this.tocStart.length,
+      md.lastIndexOf(this.tocEnd),
+    );
+    const result = toc.replace(/\[(.*)\]\(.*\)/g, '$1');
+    return result;
   }
 
   /**
-   * Retrieve the collapse css class based on the current filter
+   * Strips the table of contents from a markdown string
    *
-   * @returns The resulting class as string
+   * @returns The resulting string
    */
-  public collapseClass(): string {
-    if (!this.filter.optionIsEmpty(OptionType.documentation)) {
-      return 'show';
-    }
-    return '';
+  public stripToc(md: string): string {
+    return md.substring(md.lastIndexOf(this.tocEnd));
   }
 }
diff --git a/src/app/notes/notes.effects.ts b/src/app/notes/notes.effects.ts
index afff4f69..0a6fc2dc 100644
--- a/src/app/notes/notes.effects.ts
+++ b/src/app/notes/notes.effects.ts
@@ -4,7 +4,7 @@ import { of } from 'rxjs';
 import { catchError, exhaustMap, map } from 'rxjs/operators';
 import { ActionTypes, DoFilterSuccess, DoFilter, Failed, GetNotesSuccess } from './notes.actions';
 import { NotesService } from './notes.service';
-import { Note } from '@app/shared/model/notes.model';
+import { Kep } from '@app/shared/model/notes.model';
 import { OptionType } from '@app/shared/model/options.model';
 import { LoggerService } from '@shared/services/logger.service';
 
@@ -15,7 +15,7 @@ export class NotesEffects {
     ofType(ActionTypes.GetNotes),
     exhaustMap(() =>
       this.notesService.getNotes().pipe(
-        map((notes: Note[]) => {
+        map((notes: Kep[]) => {
           this.logger.debug('[Notes Effects:GetNotes] SUCCESS');
           return new GetNotesSuccess(notes);
         }),
@@ -40,62 +40,38 @@ export class NotesEffects {
       };
     }),
     exhaustMap(data => {
-      const filteredNotes: Set<Note> = new Set();
+      const filteredKeps: Set<Kep> = new Set();
       const filter = data.filter;
 
-      for (const note of data.notes) {
-        // Filter the release versions
-        if (
-          !filter.has(OptionType.releaseVersions, note.release_version) &&
-          (!filter.optionIsEmpty(OptionType.releaseVersions) ||
-            filter.isPreRelease(note.release_version))
-        ) {
-          // Wrong release version set, it will not be added
-          continue;
-        }
-
+      for (const kep of data.notes) {
         // If the filter is empty (this ignores the releaseVersions field),
         // show the note
         if (filter.isEmpty()) {
-          filteredNotes.add(note);
+          filteredKeps.add(kep);
           continue;
         }
 
         // Filter all regular option types
-        if (filter.hasAny(OptionType.areas, note.areas)) {
-          filteredNotes.add(note);
-          continue;
-        }
-        if (filter.hasAny(OptionType.kinds, note.kinds)) {
-          filteredNotes.add(note);
-          continue;
-        }
-        if (filter.hasAny(OptionType.sigs, note.sigs)) {
-          filteredNotes.add(note);
+        if (filter.has(OptionType.owningSigs, kep.owningSig)) {
+          filteredKeps.add(kep);
           continue;
         }
-        if (
-          note.documentation &&
-          filter.hasAny(
-            OptionType.documentation,
-            note.documentation.map(x => x.type.toString()),
-          )
-        ) {
-          filteredNotes.add(note);
+        if (filter.hasAny(OptionType.participatingSigs, kep.participatingSigs)) {
+          filteredKeps.add(kep);
           continue;
         }
 
         // Now lazily filter every string based field via the text based filter
         if (filter.text.length > 0) {
-          for (const key of Object.keys(note)) {
+          for (const key of Object.keys(kep)) {
             if (
-              typeof note[key] === 'string' &&
-              note[key]
+              typeof kep[key] === 'string' &&
+              kep[key]
                 .toUpperCase()
                 .trim()
                 .includes(filter.text.toUpperCase().trim())
             ) {
-              filteredNotes.add(note);
+              filteredKeps.add(kep);
               break;
             }
           }
@@ -103,7 +79,7 @@ export class NotesEffects {
       }
 
       this.logger.debug('[Notes Effects:DoFilter] SUCCESS (filtered)');
-      return of(new DoFilterSuccess([...filteredNotes]));
+      return of(new DoFilterSuccess([...filteredKeps]));
     }),
   );
 
diff --git a/src/app/notes/notes.reducer.ts b/src/app/notes/notes.reducer.ts
index d68ced93..a90a93af 100644
--- a/src/app/notes/notes.reducer.ts
+++ b/src/app/notes/notes.reducer.ts
@@ -1,12 +1,12 @@
 import { createSelector } from '@ngrx/store';
 import { ActionTypes, NotesAction } from './notes.actions';
-import { Note } from '@app/shared/model/notes.model';
+import { Kep } from '@app/shared/model/notes.model';
 import { State as RootState } from '@app/app.reducer';
 
 export interface State {
   error: string | null;
-  filteredNotes: Note[];
-  notes: Note[];
+  filteredNotes: Kep[];
+  notes: Kep[];
 }
 
 export const initialState: State = {
diff --git a/src/app/notes/notes.service.ts b/src/app/notes/notes.service.ts
index 5f2edd51..1ddb10af 100644
--- a/src/app/notes/notes.service.ts
+++ b/src/app/notes/notes.service.ts
@@ -4,7 +4,7 @@ import { HttpClient } from '@angular/common/http';
 import { Observable, forkJoin } from 'rxjs';
 import { map } from 'rxjs/operators';
 
-import { Note } from '@app/shared/model/notes.model';
+import { Kep } from '@app/shared/model/notes.model';
 import { LoggerService } from '@shared/services/logger.service';
 import { assets } from '@env/assets';
 
@@ -19,7 +19,7 @@ export class NotesService {
    *
    * @returns The NoteList as observable
    */
-  getNotes(): Observable<Note[]> {
+  getNotes(): Observable<Kep[]> {
     this.logger.debug(`Gathering notes from ${assets.length} assets`);
 
     const observables = [];
@@ -35,12 +35,14 @@ export class NotesService {
    *
    * @returns The Note list
    */
-  toNoteList(jsonArray: any[]): Note[] {
+  toNoteList(jsonArray: any[]): Kep[] {
     const list = [];
 
     for (let i = 0, len = jsonArray.length; i < len; i++) {
-      for (const value of Object.values(jsonArray[i])) {
-        list.push(value);
+      const data = jsonArray[i];
+      for (const key of Object.keys(data)) {
+        data[key].id = key;
+        list.push(data[key]);
       }
     }
 
diff --git a/src/app/shared/model/filter.model.ts b/src/app/shared/model/filter.model.ts
index 83deb482..a39bec76 100644
--- a/src/app/shared/model/filter.model.ts
+++ b/src/app/shared/model/filter.model.ts
@@ -30,8 +30,8 @@ export class Filter extends Options {
       return false;
     }
     let empty = true;
-    this.data.forEach((value: OptionSet, key: string) => {
-      if (key !== OptionType.releaseVersions && value.size > 0) {
+    this.data.forEach((value: OptionSet, _: string) => {
+      if (value.size > 0) {
         empty = false;
       }
     });
diff --git a/src/app/shared/model/notes.model.mock.ts b/src/app/shared/model/notes.model.mock.ts
index 8615b57b..ce088be1 100644
--- a/src/app/shared/model/notes.model.mock.ts
+++ b/src/app/shared/model/notes.model.mock.ts
@@ -1,27 +1,21 @@
-import { DocType, Documentation, Note } from './notes.model';
+import { Kep } from './notes.model';
 
-export const documentationMock: Documentation = {
-  description: 'Relevant information',
-  url: 'https://kubernetes.io/docs/concepts/workloads/pods/pod',
-  type: DocType.official,
-};
-
-export const notesMock: Note[] = [
+export const notesMock: Kep[] = [
   {
-    commit: 'commit',
-    text: 'text',
-    markdown: 'markdown',
-    author: 'author',
-    authorUrl: 'authorUrl',
-    prUrl: 'prUrl',
-    prNumber: 1,
-    areas: ['area'],
-    kinds: ['kind'],
-    sigs: ['sig'],
-    feature: true,
-    duplicate: true,
-    action_required: true,
-    release_version: 'release_version',
-    documentation: [documentationMock],
+    id: 'id',
+    title: 'title',
+    authors: [],
+    owningSig: 'sig-cloud-provider',
+    participatingSigs: ['sig-node'],
+    reviewers: [],
+    approvers: [],
+    editor: '',
+    creationDate: new Date(),
+    lastUpdated: new Date(),
+    status: '',
+    seeAlso: [],
+    replaces: [],
+    supersededBy: [],
+    markdown: '',
   },
 ];
diff --git a/src/app/shared/model/notes.model.ts b/src/app/shared/model/notes.model.ts
index 4f6f3f70..e02383be 100644
--- a/src/app/shared/model/notes.model.ts
+++ b/src/app/shared/model/notes.model.ts
@@ -1,29 +1,17 @@
-export interface Note {
-  commit: string;
-  text: string;
+export interface Kep {
+  id: string;
+  title: string;
+  authors: string[];
+  owningSig: string;
+  participatingSigs: string[];
+  reviewers: string[];
+  approvers: string[];
+  editor: string;
+  creationDate: Date;
+  lastUpdated: Date;
+  status: string;
+  seeAlso: string[];
+  replaces: string[];
+  supersededBy: string[];
   markdown: string;
-  author: string;
-  authorUrl: string;
-  prUrl: string;
-  prNumber: number;
-  areas: string[];
-  kinds: string[];
-  sigs: string[];
-  feature: boolean;
-  duplicate: boolean;
-  action_required: boolean;
-  release_version: string;
-  documentation: Documentation[];
-}
-
-export interface Documentation {
-  description: string | null;
-  url: string;
-  type: DocType;
-}
-
-export enum DocType {
-  external = 1,
-  kep,
-  official,
 }
diff --git a/src/app/shared/model/options.model.ts b/src/app/shared/model/options.model.ts
index b80b4ebb..ec48f0e9 100644
--- a/src/app/shared/model/options.model.ts
+++ b/src/app/shared/model/options.model.ts
@@ -2,11 +2,8 @@
  * All available option types
  */
 export enum OptionType {
-  areas = 'areas',
-  kinds = 'kinds',
-  releaseVersions = 'releaseVersions',
-  sigs = 'sigs',
-  documentation = 'documentation',
+  participatingSigs = 'participatingSigs',
+  owningSigs = 'owningSig',
 }
 
 /**
@@ -27,11 +24,8 @@ export class Options {
    * The private data store
    */
   private store: OptionData = new Map([
-    [OptionType.areas, new Set()],
-    [OptionType.kinds, new Set()],
-    [OptionType.releaseVersions, new Set()],
-    [OptionType.sigs, new Set()],
-    [OptionType.documentation, new Set()],
+    [OptionType.owningSigs, new Set()],
+    [OptionType.participatingSigs, new Set()],
   ]);
 
   /**
@@ -82,10 +76,7 @@ export class Options {
    * Sort the internal data structures by its defined logical order
    */
   private sort() {
-    [OptionType.areas, OptionType.kinds, OptionType.sigs, OptionType.documentation].forEach(x =>
-      this.sort_set(x),
-    );
-    this.sort_set(OptionType.releaseVersions, (a, b) => (a < b ? 1 : -1));
+    [OptionType.owningSigs, OptionType.participatingSigs].forEach(x => this.sort_set(x));
   }
 
   /**
diff --git a/src/assets/keps.json b/src/assets/keps.json
new file mode 100644
index 00000000..92879825
--- /dev/null
+++ b/src/assets/keps.json
@@ -0,0 +1,3571 @@
+[
+  {
+    "id": "5322b8bc501906188172ab1818558107",
+    "title": "aws-k8s-tester",
+    "authors": ["@gyuho"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": null,
+    "reviewers": ["@d-nishi", "@shyamjvs"],
+    "approvers": ["@d-nishi", "@shyamjvs"],
+    "editor": "TBD",
+    "creationDate": "2018-11-26",
+    "lastUpdated": "2018-11-29",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# aws-k8s-tester - kubetest plugin for AWS and EKS\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Kubernetes E2E test workflow: upstream, Prod EKS builds](#kubernetes-e2e-test-workflow-upstream-prod-eks-builds)\n    - [Sub-project E2E test workflow: upstream, ALB Ingress Controller](#sub-project-e2e-test-workflow-upstream-alb-ingress-controller)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nAll e2e tests maintained by AWS uses `aws-k8s-tester` as a plugin to kubetest. `aws-k8s-tester` runs various Kubernetes testing operations (e.g. create a temporary EKS cluster) mainly to implement [`kubernetes/test-infra/kubetest.deployer`](https://github.com/kubernetes/test-infra/blob/40b4010f8e38582a5786adedd4e04cf4e1fc5a36/kubetest/main.go#L222-L229) interface.\n\n## Motivation\n\nMany AWS tests are run by community projects such as [kops](https://github.com/kubernetes/kops), which does not test EKS. Does EKS Service check out-of-range NodePort? Does EKS Service prevent NodePort collisions? Can EKS support 5,000 nodes? There were thousands more to cover. Incomplete test coverage feeds production issues: one component failure may evolve into cluster-wide outage, Kubernetes CSI driver might be incompatible with Amazon EBS, customers may experience scalability problems from untested features, etc. Complete test coverage will unearth such issues beforehand, which leads to better customer experience. This work alone will make a huge impact on improving EKS reliability and its release cadence.\n\n### Goals\n\nThe following key features are in scope:\n\n* EKS cluster creation/teardown\n* Test open-source Kubernetes distribution (e.g. wrap kubeadm to run Kubernetes e2e tests)\n* Create AWS resources to test AWS sub-projects (e.g. create EC2 instances to test CSI driver)\n\nThese are the key design principles that guides EKS testing development:\n\n- *Platform Uniformity*: EKS provides a native Kubernetes experience. To keep two platforms in sync, EKS must be tested with upstream Kubernetes. Whenever a new feature is added to Kubernetes, customer should assume it will be tested against EKS. For example, encryption provider feature was added to Kubernetes 1.10 as an alpha feature, but customers have to wait until it becomes a stable feature. Rigorous test coverage will enable more new features to customers at earliest.\n- *Maximize Productivity*: Test automation is essential to EKS development productivity at scale. EKS team should do the minimum amount of work possible to upgrade Kubernetes (including etcd). To this end, pre-prod and prod EKS builds will be continuously tested for every PR created in upstream Kubernetes. For example, etcd client upgrade in API server must be tested against EKS control plane components. If the upgrade test fails EKS, we should block the change.\n- *Transparency for Community*: We want to contribute EKS tests to upstream and make test results visible to the whole communities. Users should be able to see how EKS performs with 5,000 worker nodes and compare it with other providers, just by looking at upstream performance dashboard.\n\n### Non-Goals\n\n* The project does not replace kops or kubeadm.\n* This project is only meant for testing.\n\n## Proposal\n\n### User Stories\n\n#### Kubernetes E2E test workflow: upstream, Prod EKS builds\n\nEKS uses `aws-k8s-tester` as a plugin to kubetest. `aws-k8s-tester` is a broker that creates and deletes AWS resources on behalf of kubetest, connects to pre-prod EKS clusters, reports test results back to dashboards, etc. Every upstream change will be tested against EKS cluster.\n\nFigure 1 shows how AWS would run Kubernetes e2e tests inside EKS (e.g. ci-kubernetes-e2e-aws-eks).\n\n![20181126-aws-k8s-tester-figure-01](20181126-aws-k8s-tester-figure-01.png)\n\n#### Sub-project E2E test workflow: upstream, ALB Ingress Controller\n\nLet's take ALB Ingress Controller for example. Since Kubernetes cluster is a prerequisite to ALB Ingress Controller, `aws-k8s-tester` first creates EKS cluster. Then ALB Ingress Controller plug-in deploys and creates Ingress objects, with sample web server and client. awstester is configured through YAML rather than POSIX flags. This makes it easier to implement sub-project add-ons (e.g. add “alb-ingress-controller” field to set up ingress add-on). Cluster status, ingress controller states, and testing results are persisted to disk for status report and debugging purposes.\n\nFigure 2 shows how `aws-k8s-tester` plugin creates and tests ALB Ingress Controller.\n\n![20181126-aws-k8s-tester-figure-02](20181126-aws-k8s-tester-figure-02.png)\n\n### Implementation Details/Notes/Constraints\n\nWe implement kubetest plugin, out-of-tree and provided as a single binary file. Separate code base speeds up development and makes dependency management easier. For example, kops in kubetest uses AWS SDK [v1.12.53](https://github.com/aws/aws-sdk-go/releases/tag/v1.12.53), which was released at December 2017. Upgrading SDK to latest would break existing kops. Packaging everything in a separate binary gives us freedom to choose whatever SDK version we need.\n\n### Risks and Mitigations\n\n* *“aws-k8s-tester” creates a key-pair using EC2 API. Is the private key safely managed?* Each test run creates a temporary key pair and stores the private key on disk. The private key is used to SSH access into Kubernetes worker nodes and read service logs from the EC2 instance. “aws-k8s-tester” safely [deletes the private key on disk](https://github.com/aws/aws-k8s-tester/blob/cde0484f0ae167d8831442a48b4b5e447481af45/internal/ec2/key_pair.go#L65) and [destroys all associated AWS resources](https://github.com/aws/aws-k8s-tester/blob/cde0484f0ae167d8831442a48b4b5e447481af45/internal/ec2/key_pair.go#L71-L73), whether the test completes or get interrupted. For instance, when it deletes the key pair object from EC2, the public key is also deleted, which means the local private key has no use for any threat.\n* *Does “aws-k8s-tester” store any sensitive information?* “aws-k8s-tester” maintains a test cluster state in [`ClusterState`](https://godoc.org/github.com/aws/awstester/eksconfig#ClusterState), which is periodically synced to local disk and S3. It does not contain any sensitive data such as private key blobs.\n* *Upstream Kubernetes test-infra team mounts our AWS test credential to their Prow cluster. Can anyone access the credential?* Upstream Prow cluster schedules all open-source Kubernetes test runs. In order to test EKS from upstream Kubernetes, AWS credential must be accessible from each test job. Currently, it is mounted as a Secret object (https://kubernetes.io/docs/concepts/configuration/secret/) in upstream Prow cluster. Which means our AWS credential is still stored as base64-encoded plaintext in etcd. Then, there are two ways to access this data. One is to read from Prow testing pod (see [test-infra/PR#9940](https://github.com/kubernetes/test-infra/pull/9940/files)). In theory, any test job has access to “eks-aws-credentials” secret object, thus can maliciously mount it to steal the credential. In practice, every single job needs an approval before it runs any tests. So, if anybody tries to exploit the credential, the change should be rejected beforehand. Two, read the non-encrypted credential data from etcd. This is unlikely as well. We can safely assume that Google GKE deploys etcd in a trusted environment, where the access is restricted to Google test-infra team. See https://kubernetes.io/docs/concepts/configuration/secret/#risks for more.\n\n## Graduation Criteria\n\n`aws-k8s-tester` will be considered successful when it is used by the majority of AWS Kubernetes e2e tests.\n\n## Implementation History\n\n* Initial integration with upstream has been tracked \n* Initial proposal to SIG 2018-11-26\n* Initial KEP draft 2018-11-26\n"
+  },
+  {
+    "id": "70115a3425eb1e91e93e2cbccbcbeaae",
+    "title": "aws-ebs-csi-driver",
+    "authors": ["@leakingtapan"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": null,
+    "reviewers": ["@d-nishi", "@jsafrane"],
+    "approvers": ["@d-nishi", "@jsafrane"],
+    "editor": "TBD",
+    "creationDate": "2018-11-27",
+    "lastUpdated": "2019-01-27",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# AWS Elastic Block Store (EBS) CSI Driver\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Static Provisioning](#static-provisioning)\n    - [Dynamic Provisioning](#dynamic-provisioning)\n    - [Volume Scheduling](#volume-scheduling)\n    - [Mount Options](#mount-options)\n    - [Raw Block Volume](#raw-block-volume)\n    - [Offline Volume Resizing](#offline-volume-resizing)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Upgrade/Downgrade Process](#upgradedowngrade-process)\n  - [Upgrade](#upgrade)\n  - [Downgrade](#downgrade)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\nAWS EBS CSI Driver implements [Container Storage Interface](https://github.com/container-storage-interface/spec/tree/master) which is the standard of storage interface for container. It provides the same in-tree AWS EBS plugin features including volume creation, volume attachment, volume mounting and volume scheduling. It is also configurable on what is the EBS volume type to create, what is the file system file should be formatted, which KMS key to use to create encrypted volume, etc.\n\n## Motivation\nSimilar to CNI plugins, AWS EBS CSI driver will be a stand alone plugin that lives out-of-tree of kuberenetes. Being out-of-tree, it will be benefit from being modularized, maintained and optimized without affecting kubernetes core code base. Aside from those benefits, it could also be consumed by other container orchestrators such as ECS.\n\n### Goals\nAWS EBS CSI driver will provide similar user experience as in-tree EBS plugin:\n* An application developer will not notice any difference in the operation of EBS CSI driver versus the in-tree volume plugin. His/Her workflow will stay the same as before.\n* An infrastructure operator needs to deploy/upgrade the driver and create/update storageclass to let the driver to manage underlying storage backend. The storageclass need not be updated if the name of the csi-driver referenced does not change.\n\nSince EBS CSI Driver is out-of-tree implementation that comes outside of kuberenetes distrubtion, documentations will be provided on how to install, use and upgrade the driver.\n\nList of driver features include volume creation/deletion, volume attach/detach, volume mount/unmount, volume scheduling, create volume configurations, volume snapshotting, mount options, raw block volume, etc.\n\n### Non-Goals\n* Supporting non AWS block storage\n* Supporting other AWS storage serivces such as Dynamodb, S3, etc.\n\n## Proposal\n\n### User Stories\n\n#### Static Provisioning\nOperator creates a pre-created EBS volume on AWS and a CSI PV that refers the EBS volume on cluster. Developer creates PVC and a Pod that uses the PVC. Then developer deploys the Pod during which time the PV will be attached to container inside Pod after PVC bonds to PV successfully.\n\n#### Dynamic Provisioning\nOperator creates a storage class that defines EBS CSI driver as provisioner. Developer creates PVC and a Pod that uses the PVC. A new CSI PV will be created dynamically and be bound to the defined PVC. Finally, the PV will be attached to container inside Pod.\n\n#### Volume Scheduling\nOperation creates StorageClass with  volumeBindingMode = WaitForFirstConsumer. When developer deploys a Pod that has PVC that is trying to claim for a PV, a new PV will be created, attached, formatted and mounted inside Pod\u0026#39;s container by the EBS CSI driver. Topology information provided by EBS CSI driver will be used during Pod scheduling to guarantee that both Pod and volume are collocated in the same availability zone.\n\n#### Mount Options\nOperator creates a storage class that defines mount option of the persistence volume. When a PV is dynamically provisioned, the volume will be mounted inside container using the provided mount option.\nOperator creates a PV which is backed by a EBS volume manually. The PV spec defines the mount option (eg. ro) of the volume. When the PV is consumed by the application and the Pod is running, the volume will be mounted inside container with the given mount option.\n\n#### Raw Block Volume\nOperator creates PV or PVC with `volumeMode: Block`. When application consumes the volume, it is mounted inside container as raw device (eg. /dev/sdba).\n\n#### Offline Volume Resizing\nOperator enables the allowVolumeExpansion feature in storageclass. When there is no Pod consuming the volume and user resizes the volume by editing the requested storage size in PVC, the volume got resized by the driver with the given new size.\n\n### Risks and Mitigations\n* *Information disclosure* - AWS EBS CSI driver requires permission to perform AWS operations on behalf of the user. The CSI driver will not log any of the user credentials. We will also provide the user with policies that limit the access of the driver to required AWS services.\n* *Escalation of Privileges* - Since EBS CSI driver is formatting and mounting volumes, it requires root privilege to permform the operations. So that driver will have higher privilege than other containers in the cluster. The driver will not execute random commands provided by untrusted user. All of its interfaces are only provided for kuberenetes system components to interact with. The driver will also validate requests to make sure it aligns with its assumption.\n\n## Graduation Criteria\n* Static provisioning is implemented.\n* Dynamic provisioning is implemented.\n* Volume scheduling is implemented.\n* Mount options is implmented.\n* Raw block volume is implemented .\n* Offline volume resizing is implemented.\n* Integration test is implemented and integrated with Prow and Testgrid.\n* E2E tests are implemented and integrated with Prow and Testgrid.\n\n## Upgrade/Downgrade Process\nThis assumes user is already using some version of the driver.\n\n### Upgrade\nThis assumes user is already using Kubernetes 1.13 cluster. Otherwise, the existing cluster needs to be upgraded to 1.13+ in order to install the driver. Formal cluster upgrade process should be followed for upgrading cluster.\n\nDriver upgrade should be performed one version at a time. This means, if the current driver version is 0.1, it can be upgraded to version 0.2 by following the upgrade process. And if the driver version that is required to upgrade to is 0.3, it should be upgraded to 0.2 first.\n\nTo upgrade the driver, perform following steps:\n1. Delete the old driver controller service and node service along with other resources including cluster roles, cluster role bindings and service accounts.\n1. Deploy the new driver controller service and node service along with other resources including cluster roles, cluster role bindings and service accounts.\n\n### Downgrade\nSimilar to driver upgrade, driver downgrade should be performed one version at a time.\n\nTo downgrade the driver, perform following steps:\n1. Delete the old driver controller service and node service along with other resources including cluster roles, cluster role bindings and service accounts.\n1. Deploy the new driver controller service and node service along with other resources inclluding cluster roles, cluster role bindings and service accounts.\n\n## Implementation History\n* 2018-11-26 Initial proposal to SIG\n* 2018-11-26 Initial KEP draft\n* 2018-12-03 Alpha release with kuberentes 1.13\n* 2018-03-25 Beta release with kubernetes 1.14\n\n"
+  },
+  {
+    "id": "b61c081bb306f943de326cf3e2216fc5",
+    "title": "Custom endpoints support for AWS Cloud Provider",
+    "authors": ["@micahhausler"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": ["sig-cloud-provider"],
+    "reviewers": ["@justinsb", "@mcrute"],
+    "approvers": ["@justinsb"],
+    "editor": "@micahhausler",
+    "creationDate": "2019-01-28",
+    "lastUpdated": "2019-01-28",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Custom endpoint support for AWS Cloud Provider\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nAWS service APIs typically operate at fixed domain name endpoints, but in\ncertain cases may function at a different endpoint than the AWS SDKs are aware\nof. The AWS Cloud Provider should support these custom endpoints.\n\n## Motivation\n\nBeing able to support custom endpoints enables Kubernetes users to use alternate\nimplementations of AWS APIs such as [Eucalyptus][] and alernate AWS endpoints\nfor AWS Service APIs to support [AWS PrivateLink][]. AWS PrivateLink allows AWS users to\nensure their AWS API calls do not transit the public internet.\n\n[Eucalyptus]: https://www.eucalyptus.cloud/\n[AWS PrivateLink]: https://docs.aws.amazon.com/vpc/latest/userguide/vpce-interface.html\n\n### Goals\n\n- Allow Kubernetes Cloud Controller to use custom endpoints for AWS services\n- Extend existing CloudConfig INI file to specify endpoints\n- Allow Kubelet to use custom endpoints for ECR credential retrieval\n\n### Non-Goals\n\n- Multi-region AWS cloud provider support\n\n## Proposal\n\n\n## Graduation Criteria\n\nSupport for custom endpoints in both the kubelet and cloud controller.\n\n## Implementation History\n\n- Initial CloudController implementation [#72245][]\n\n[#72245]: https://github.com/kubernetes/kubernetes/pull/72245/files\n"
+  },
+  {
+    "id": "d61087a8ebad46006d896250d5255547",
+    "title": "graduate-aws-nlb-to-beta",
+    "authors": ["@M00nF1sh"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": null,
+    "reviewers": ["@dnishi"],
+    "approvers": ["@justinsb", "@dnishi"],
+    "editor": "TBD",
+    "creationDate": "2019-05-01",
+    "lastUpdated": "2019-05-01",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Graduate AWS Network Load Balancer Support to beta\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n- [Design](#design)\n  - [Test Plan](#test-plan)\n    - [Needed Tests](#needed-tests)\n  - [Graduation Criteria](#graduation-criteria)\n- [Proposed roadmap](#proposed-roadmap)\n  - [1.15](#115)\n  - [1.16](#116)\n  - [1.18](#118)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\n[AWS Network Load Balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/introduction.html) is an L4 Load Balancer on AWS platform, that can be used to declare a Kubernetes Service with type: LoadBalancer.\n\n## Motivation\n\n[AWS Network Load Balancer](https://aws.amazon.com/blogs/opensource/network-load-balancer-support-in-kubernetes-1-9/) has been supported in Kubernetes as Alpha feature since v1.9. Since then, the code and API has been stabilized. Therefore, we would like to graduate NLB support from Alpha to Beta.\n\n### Goals\n\n* Promote AWS Network Load Balancer support to beta version.\n\n## Proposal\n\n### User Stories\n* An application developer or infrastructure engineer who wants to use AWS Network Load balancer can declare their Kubernetes Service with type: Load Balancer for any Kubernetes clusters running in the AWS cloud. \n\n## Design\n\n### Test Plan\n\n#### Needed Tests\n\n- Add E2E tests to allow usage of NLB to declare Kubernetes Service with type: LoadBalancer.\n\n### Graduation Criteria\n- [x] Support Cross-zone Load Balancing\n- [x] Support TLS termination\n- [] Have documentation for NLB annotations\n- [ ] Have E2E test\n- [x] Have roadmap for future development\n\n## Proposed roadmap\n### 1.15\n* [ ] Graduate AWS Network Load Balancer support to beta\n### 1.16\n* [ ] Deprecate usage of AWS Classic Load Balancer as the default implementation for Service with LoadBalancerType on AWS\n* [ ] Notify users to migrate to use NLB instead\n### 1.18\n* [ ] Use AWS Network Load Balancer as the default implementation for Service with LoadBalancer Type on aws\n\n## Implementation History\n\n- AWS Network Load Balancer Support was introduced as alpha in kubernetes 1.9\n- [support cross-zone load balancing](https://github.com/kubernetes/kubernetes/pull/61064)\n- [support TLS termination](https://github.com/kubernetes/kubernetes/pull/74910)\n- [Bug fix - SecurityGroup rule removed incorrectly](https://github.com/kubernetes/kubernetes/pull/68422)\n- [Bug fix - LoadBalancerSourceRanges not working](https://github.com/kubernetes/kubernetes/pull/74692)\n"
+  },
+  {
+    "id": "20530ee12f8308c7c5823b1fc40c7d40",
+    "title": "AWS LoadBalancer Prefix",
+    "authors": ["@minherz"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": null,
+    "reviewers": ["TBD"],
+    "approvers": ["TBD"],
+    "editor": "TBD",
+    "creationDate": "2018-11-02",
+    "lastUpdated": "2018-11-02",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# AWS LoadBalancer Prefix Annotation Proposal\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories [optional]](#user-stories-optional)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Drawbacks [optional]](#drawbacks-optional)\n- [Alternatives [optional]](#alternatives-optional)\n- [Infrastructure Needed [optional]](#infrastructure-needed-optional)\n\u003c!-- /toc --\u003e\n\n## Summary\nAWS load balancer prefix annotation adds a control over the naming of the AWS ELB resources that are being generated when provisioning a Kubernetes service of type `LoadBalancer`. The current implementation provisions AWS ELB with a unique name based on the resource UID. The resulted unpredicted name makes it impossible to integrate the provisioning with existing IAM policies in situations when these two operations are controlled by two different groups. For example, IAM policies are defined and controlled by InfoSec team while provisioning of resources is under CloudOps team. The AWS IAM policies allow definition when only a prefix of the resource identifier is known. Using Kubernetes service with this annotation when it is provisioned in AWS, will allow an integration with existing IAM policies.\n\n## Motivation\nCurrent way of provisioning load balancer (for a Kubernetes service of the type `LoadBalancer`) is to use the service's UID and to follow Cloud  naming conventions for load balancers (for AWS it is a 32 character sequence of alphanumeric characters or hyphens that cannot begin or end with hypen [link1](https://docs.aws.amazon.com/elasticloadbalancing/2012-06-01/APIReference/API_CreateLoadBalancer.html), [link2](https://docs.aws.amazon.com/cli/latest/reference/elbv2/create-load-balancer.html)). When it is provisioned on AWS account with predefined IAM policies that limit access to ELB resources using wildcarded paths (IAM identifiers), the Kubernetes service cannot be provisioned. Providing a way to define a short known prefix to ELB resource makes it possible to match IAM policies conditions regarding the resource identifiers.\n\n### Goals\n* Support provisioning of AWS ELB resources for Kubernetes services of the type `LoadBalancer` that match AWS IAM policies\n### Non-Goals\n* Provide meaningful names for AWS ELB resources generated for Kubernetes services of the type `LoadBalancer`\n\n## Proposal\n\n### User Stories [optional]\n\n### Implementation Details/Notes/Constraints [optional]\n\n### Risks and Mitigations\n\n## Graduation Criteria\n\n## Implementation History\n\n## Drawbacks [optional]\n\n## Alternatives [optional]\n\n## Infrastructure Needed [optional]\n"
+  },
+  {
+    "id": "6c12239a7d66b027d2ea3b4a1c5eb377",
+    "title": "AWS ALB Ingress Controller",
+    "authors": ["@M00nF1sh"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": null,
+    "reviewers": ["TBD", "@d-nishi"],
+    "approvers": ["TBD", "@d-nishi"],
+    "editor": "TBD",
+    "creationDate": "2018-11-27",
+    "lastUpdated": "2019-01-27",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# AWS ALB Ingress Controller\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Expose HTTP[s] API backed by kubernetes services](#expose-https-api-backed-by-kubernetes-services)\n    - [Adjust ALB settings via annotation](#adjust-alb-settings-via-annotation)\n    - [Leverage WAF \u0026amp; Cognito](#leverage-waf--cognito)\n    - [Sharing single ALB among Ingresses across namespace](#sharing-single-alb-among-ingresses-across-namespace)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis proposal introduces [AWS ALB Ingress Controller](https://github.com/kubernetes-sigs/aws-alb-ingress-controller/) as Ingress controller for kubernetes cluster on AWS. Which use [Amazon Elastic Load Balancing Application Load Balancer](https://aws.amazon.com/elasticloadbalancing/features/#Details_for_Elastic_Load_Balancing_Products)(ALB) to fulfill [Ingress resources](https://kubernetes.io/docs/concepts/services-networking/ingress/), and provides integration with various AWS services.\n\n## Motivation\n\nIn order for the Ingress resource to work, the cluster must have an Ingress controller runnings. However, existing Ingress controllers like [nginx](https://github.com/kubernetes/ingress-nginx/blob/master/README.md) didn't take advantage of native AWS features.\nAWS ALB Ingress Controller aims to enhance Ingress resource on AWS by leveraging rich feature set of ALB, such as host/path based routing, TLS termination, WebSockets, HTTP/2. Also, it will provide close integration with other AWS services such as WAF(web application firewall) and Cognito.\n\n### Goals\n\n* Support running multiple Ingress controllers in cluster\n* Support portable Ingress resource(no annotations)\n* Support leverage feature set of ALB via custom annotations\n* Support integration with WAF\n* Support integration with Cognito\n* Support multiple ACM certificates via annotation or IngressTLS.\n\n### Non-Goals\n\n* This project does not replacing nginx ingress controller\n\n## Proposal\n\n### User Stories\n\n#### Expose HTTP[s] API backed by kubernetes services\nDevelopers create an Ingress resources to specify rules for how to routing HTTP[s] traffic to different services.\nAWS ALB Ingress Controller will monitor such Ingress resources and create ALB and other necessary supporting AWS resources to match the Ingress resource specification.\n\n#### Adjust ALB settings via annotation\nDevelopers specifies custom annotations on their Ingress resource to adjust ALB settings, such as enable deletion protection, enable access logs to specific S3 bucket.\n\n#### Leverage WAF \u0026 Cognito\nDevelopers specifies custom annotations on their Ingress resource to denote WAF and Cognito integrations. Which provides web application firewall and authentication support for their exposed API.\n\n#### Sharing single ALB among Ingresses across namespace\nDevelopers from different teams create Ingress resources in different namespaces which route traffic to services within their own namespace. However, an single ALB is shared from these Ingresses to expose a single DNS name for customers.\n\n## Graduation Criteria\n\n* AWS ALB Ingress Controller is widely used as Ingress controller for kubernetes clusters on AWS\n* Comprehensive documentation about features and use cases.\n* E2E tests are implemented and integrated with Prow and Testgrid.\n\n## Implementation History\n- [community#2841](https://github.com/kubernetes/community/pull/2841) Design proposal\n- [aws-alb-ingress-controller#738](https://github.com/kubernetes-sigs/aws-alb-ingress-controller/pull/738) First stable release: v1.0.0\n- 2018-12-03 Alpha release with kuberentes 1.13\n- 2018-03-25 Beta release with kubernetes 1.14 (scheduled)\n"
+  },
+  {
+    "id": "a0ae7803780c8b887219b15ed6c7000d",
+    "title": "Apply",
+    "authors": ["@lavalamp"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["sig-api-machinery", "sig-cli"],
+    "reviewers": ["@pwittrock", "@erictune"],
+    "approvers": ["@bgrant0607"],
+    "editor": "TBD",
+    "creationDate": "2018-03-28",
+    "lastUpdated": "2018-03-28",
+    "status": "implementable",
+    "seeAlso": ["n/a"],
+    "replaces": ["n/a"],
+    "supersededBy": ["n/a"],
+    "markdown": "\n# Apply\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n    - [API Topology](#api-topology)\n      - [Lists](#lists)\n      - [Maps and structs](#maps-and-structs)\n    - [Kubectl](#kubectl)\n      - [Server-side Apply](#server-side-apply)\n    - [Status Wiping](#status-wiping)\n      - [Current Behavior](#current-behavior)\n      - [Proposed Change](#proposed-change)\n      - [Alternatives](#alternatives)\n      - [Implementation History](#implementation-history)\n  - [Risks and Mitigations](#risks-and-mitigations)\n  - [Testing Plan](#testing-plan)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history-1)\n- [Drawbacks](#drawbacks)\n- [Alternatives](#alternatives-1)\n\u003c!-- /toc --\u003e\n\n## Summary\n\n`kubectl apply` is a core part of the Kubernetes config workflow, but it is\nbuggy and hard to fix. This functionality will be regularized and moved to the\ncontrol plane.\n\n## Motivation\n\nExample problems today:\n\n* User does POST, then changes something and applies: surprise!\n* User does an apply, then `kubectl edit`, then applies again: surprise!\n* User does GET, edits locally, then apply: surprise!\n* User tweaks some annotations, then applies: surprise!\n* Alice applies something, then Bob applies something: surprise!\n\nWhy can't a smaller change fix the problems? Why hasn't it already been fixed?\n\n* Too many components need to change to deliver a fix\n* Organic evolution and lack of systematic approach\n  * It is hard to make fixes that cohere instead of interfere without a clear model of the feature\n* Lack of API support meant client-side implementation\n  * The client sends a PATCH to the server, which necessitated strategic merge patch--as no patch format conveniently captures the data type that is actually needed.\n  * Tactical errors: SMP was not easy to version, fixing anything required client and server changes and a 2 release deprecation period.\n* The implications of our schema were not understood, leading to bugs.\n  * e.g., non-positional lists, sets, undiscriminated unions, implicit context\n  * Complex and confusing defaulting behavior (e.g., Always pull policy from :latest)\n  * Non-declarative-friendly API behavior (e.g., selector updates)\n\n### Goals\n\n\"Apply\" is intended to allow users and systems to cooperatively determine the\ndesired state of an object. The resulting system should:\n\n* Be robust to changes made by other users, systems, defaulters (including mutating admission control webhooks), and object schema evolution.\n* Be agnostic about prior steps in a CI/CD system (and not require such a system).\n* Have low cognitive burden:\n  * For integrators: a single API concept supports all object types; integrators\n    have to learn one thing total, not one thing per operation per api object.\n    Client side logic should be kept to a minimum; CURL should be sufficient to\n    use the apply feature.\n  * For users: looking at a config change, it should be intuitive what the\n    system will do. The “magic” is easy to understand and invoke.\n  * Error messages should--to the extent possible--tell users why they had a\n    conflict, not just what the conflict was.\n  * Error messages should be delivered at the earliest possible point of\n    intervention.\n\nGoal: The control plane delivers a comprehensive solution.\n\nGoal: Apply can be called by non-go languages and non-kubectl clients. (e.g.,\nvia CURL.)\n\n### Non-Goals\n\n* Multi-object apply will not be changed: it remains client side for now\n* Some sources of user confusion will not be addressed:\n  * Changing the name field makes a new object rather than renaming an existing object\n  * Changing fields that can’t really be changed (e.g., Service type).\n\n## Proposal\n\n(Please note that when this KEP was started, the KEP process was much less well\ndefined and we have been treating this as a requirements / mission statement\ndocument; KEPs have evolved into more than that.)\n\nA brief list of the changes:\n\n* Apply will be moved to the control plane.\n  * The [original design](https://goo.gl/UbCRuf) is in a google doc; joining the\n    kubernetes-dev or kubernetes-announce list will grant permission to see it.\n    Since then, the implementation has changed so this may be useful for\n    historical understanding. The test cases and examples there are still valid.\n  * Additionally, readable in the same way, is the [original design for structured diff and merge](https://goo.gl/nRZVWL);\n    we found in practice a better mechanism for our needs (tracking field\n    managers) but the formalization of our schema from that document is still\n    correct.\n* Apply is invoked by sending a certain Content-Type with the verb PATCH.\n* Instead of using a last-applied annotation, the control plane will track a\n  \"manager\" for every field.\n* Apply is for users and/or ci/cd systems. We modify the POST, PUT (and\n  non-apply PATCH) verbs so that when controllers or other systems make changes\n  to an object, they are made \"managers\" of the fields they change.\n* The things our \"Go IDL\" describes are formalized: [structured merge and diff](https://github.com/kubernetes-sigs/structured-merge-diff)\n* Existing Go IDL files will be fixed (e.g., by [fixing the directives](https://github.com/kubernetes/kubernetes/pull/70100/files))\n* Dry-run will be implemented on control plane verbs (POST, PUT, PATCH).\n  * Admission webhooks will have their API appended accordingly.\n* An upgrade path will be implemented so that version skew between kubectl and\n  the control plane will not have disastrous results.\n\nThe linked documents should be read for a more complete picture.\n\n### Implementation Details/Notes/Constraints [optional]\n\n(TODO: update this section with current design)\n\n#### API Topology\n\nServer-side apply has to understand the topology of the objects in order to make\nvalid merging decisions. In order to reach that goal, some new Go markers, as\nwell as OpenAPI extensions have been created:\n\n##### Lists\n\nLists can behave in mostly 3 different ways depending on what their actual semantic\nis. New annotations allow API authors to define this behavior.\n\n- Atomic lists: The list is owned by only one person and can only be entirely\nreplaced. This is the default for lists. It is defined either in Go IDL by\npefixing the list with `// +listType=atomic`, or in the OpenAPI\nwith `\"x-kubenetes-list-type\": \"atomic\"`.\n\n- Sets: the list is a set (it has to be of a scalar type). Items in the list\nmust appear at most once. Individual actors of the API can own individual items.\nIt is defined either in Go IDL by pefixing the list with `//\n+listType=set`, or in the OpenAPI with\n`\"x-kubenetes-list-type\": \"set\"`.\n\n- Associative lists: Kubernetes has a pattern of using lists as dictionary, with\n\"name\" being a very common key. People can now reproduce this pattern by using\n`// +listType=map`, or in the OpenAPI with `\"x-kubernetes-list-type\": \"map\"`\nalong with `\"x-kubernetes-list-map-keys\": [\"name\"]`, or `// +listMapKey=name`.\nItems of an associative lists are owned by the person who applied the item to\nthe list.\n\nFor compatibility with the existing markers, the `patchStrategy` and\n`patchMergeKey` markers are automatically used and converted to the corresponding `listType`\nand `listMapKey` if missing.\n\n##### Maps and structs\n\nMaps and structures can behave in two ways:\n- Each item in the map or field in the structure are independent from each\nother. They can be changed by different actors. This is the default behavior,\nbut can be explicitly specified with `// +mapType=granular` or `//\n+structType=granular` respectively. They map to the same openapi extension:\n`\"x-kubernetes-map-type\": \"granular\"`.\n- All the fields or item of the map are treated as one unit, we say the map/struct is\natomic. That can be specified with `// +mapType=atomic` or `//\n+structType=atomic` respectively. They map to the same openapi extension:\n`\"x-kubernetes-map-type\": \"atomic\"`.\n\n#### Kubectl\n\n##### Server-side Apply\n\nSince server-side apply is currently in the Alpha phase, it is not\nenabled by default on kubectl. To use server-side apply on servers\nwith the feature, run the command\n`kubectl apply --experimental-server-side ...`.\n\nIf the feature is not available or enabled on the server, the command\nwill fail rather than fall-back on client-side apply due to significant\nsemantical differences.\n\nAs the feature graduates to the Beta phase, the flag will be renamed to `--server-side`.\n\nThe long-term plan for this feature is to be the default apply on all\nKubernetes clusters. The semantical differences between server-side\napply and client-side apply will make a smooth roll-out difficult, so\nthe best way to achieve this has not been decided yet.\n\n#### Status Wiping\n\n##### Current Behavior\n\nRight before being persisted to etcd, resources in the apiserver undergo a preparation mechanism that is custom for every resource kind.\nIt takes care of things like incrementing object generation and status wiping.\nThis happens through [PrepareForUpdate](https://github.com/kubernetes/kubernetes/blob/bc1360ab158d524c5a7132c8dd9dc7f7e8889af1/staging/src/k8s.io/apiserver/pkg/registry/rest/update.go#L49) and [PrepareForCreate](https://github.com/kubernetes/kubernetes/blob/bc1360ab158d524c5a7132c8dd9dc7f7e8889af1/staging/src/k8s.io/apiserver/pkg/registry/rest/create_update.go#L37).\n\nThe problem status wiping at this level creates is, that when a user applies a field that gets wiped later on, it gets owned by said user.\nThe apply mechanism (FieldManager) can not know which fields get wiped for which resource and therefor can not ignore those.\n\nAdditionally ignoring status as a whole is not enough, as it should be possible to own status (and other fields) in some occasions. More conversation on this can be found in the [GitHub issue](https://github.com/kubernetes/kubernetes/issues/75564) where the problem got reported.\n\n##### Proposed Change\n\nAdd an interface that resource strategies can implement, to provide field sets affected by status wiping.\n\n```go\n# staging/src/k8s.io/apiserver/pkg/registry/rest/rest.go\n// ResetFieldsProvider is an optional interface that a strategy can implement\n// to expose a set of fields that get reset before persisting the object.\ntype ResetFieldsProvider interface {\n  // ResetFieldsFor returns a set of fields for the provided version that get reset before persisting the object.\n  // If no fieldset is defined for a version, nil is returned.\n  ResetFieldsFor(version string) *fieldpath.Set\n}\n```\n\nAdditionally, this interface is implemented by `registry.Store` which forwards it to the corresponding strategy (if applicable).\nIf `registry.Store` can not provide a field set, it returns nil.\n\nAn example implementation for the interface inside the pod strategy could be:\n\n```go\n# pkg/registry/core/pod/strategy.go\n// ResetFieldsFor returns a set of fields for the provided version that get reset before persisting the object.\n// If no fieldset is defined for a version, nil is returned.\nfunc (podStrategy) ResetFieldsFor(version string) *fieldpath.Set {\n  set, ok := resetFieldsByVersion[version]\n  if !ok {\n    return nil\n  }\n  return set\n}\n\nvar resetFieldsByVersion = map[string]*fieldpath.Set{\n  \"v1\": fieldpath.NewSet(\n    fieldpath.MakePathOrDie(\"status\"),\n  ),\n}\n```\n\nWhen creating the handlers in [installer.go](https://github.com/kubernetes/kubernetes/blob/3ff0ed46791a821cb7053c1e25192e1ecd67a6f0/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go) the current `rest.Storage` is checked to implement the `ResetFieldsProvider` interface and the result is passed to the FieldManager.\n\n```go\n# staging/src/k8s.io/apiserver/pkg/endpoints/installer.go\nvar resetFields *fieldpath.Set\nif resetFieldsProvider, isResetFieldsProvider := storage.(rest.ResetFieldsProvider); isResetFieldsProvider {\n    resetFields = resetFieldsProvider.ResetFieldsFor(a.group.GroupVersion.Version)\n}\n```\n\nWhen provided with a field set, the FieldManager strips all `resetFields` from incoming update and apply requests.\nThis causes the user/manager to not own those fields.\n\n```go\n...\nif f.resetFields != nil {\n  patchObjTyped = patchObjTyped.Remove(f.resetFields)\n}\n...\n```\n\n##### Alternatives\n\nWe looked at a way to get the fields affected by status wiping without defining them separately.\nMainly by pulling the reset logic from the strategies `PrepareForCreate` and `PrepareForUpdate` methods into a new method `ResetFields` implementing an `ObjectResetter` interface.\n\nThis approach did not work as expected, because the strategy works on internal types while the FieldManager handles external api types.\nThe conversion between the two and creating the diff was complex and would have caused a notable amount of allocations.\n\n##### Implementation History\n\n- 12/2019 [#86083](https://github.com/kubernetes/kubernetes/pull/86083) implementing a poc for the described approach\n\n### Risks and Mitigations\n\nWe used a feature branch to ensure that no partial state of this feature would\nbe in master. We developed the new \"business logic\" in a\n[separate repo](https://github.com/kubernetes-sigs/structured-merge-diff) for\nvelocity and reusability.\n\n### Testing Plan\n\nThe specific logic of apply will be tested by extensive unit tests in the\n[structured merge and diff](https://github.com/kubernetes-sigs/structured-merge-diff)\nrepo. The integration between that repo and kubernetes/kubernetes will mainly\nbe tested by integration tests in [test/integration/apiserver/apply](https://github.com/kubernetes/kubernetes/tree/master/test/integration/apiserver/apply)\nand [test/cmd](https://github.com/kubernetes/kubernetes/blob/master/test/cmd/apply.sh),\nas well as unit tests where applicable. The feature will also be enabled in the\n[alpha-features e2e test suite](https://k8s-testgrid.appspot.com/sig-release-master-blocking#gce-cos-master-alpha-features),\nwhich runs every hour and everytime someone types `/test pull-kubernetes-e2e-gce-alpha-features`\non a PR. This will ensure that the cluster can still start up and the other\nendpoints will function normally when the feature is enabled.\n\nUnit Tests in [structured merge and diff](https://github.com/kubernetes-sigs/structured-merge-diff) repo for:\n\n- [x] Merge typed objects of the same type with a schema. [link](https://github.com/kubernetes-sigs/structured-merge-diff/blob/master/typed/merge_test.go)\n- [x] Merge deduced typed objects without a schema (for CRDs). [link](https://github.com/kubernetes-sigs/structured-merge-diff/blob/master/typed/deduced_test.go)\n- [x] Convert a typed value to a field set. [link](https://github.com/kubernetes-sigs/structured-merge-diff/blob/master/typed/toset_test.go)\n- [x] Diff two typed values. [link](https://github.com/kubernetes-sigs/structured-merge-diff/blob/master/typed/symdiff_test.go)\n- [x] Validate a typed value against it's schema. [link](https://github.com/kubernetes-sigs/structured-merge-diff/blob/master/typed/validate_test.go)\n- [x] Get correct conflicts when applying. [link](https://github.com/kubernetes-sigs/structured-merge-diff/blob/master/merge/conflict_test.go)\n- [x] Apply works for deduced typed objects. [link](https://github.com/kubernetes-sigs/structured-merge-diff/blob/master/merge/deduced_test.go)\n- [x] Apply works for leaf fields with scalar values. [link](https://github.com/kubernetes-sigs/structured-merge-diff/blob/master/merge/leaf_test.go)\n- [x] Apply works for items in associative lists of scalars. [link](https://github.com/kubernetes-sigs/structured-merge-diff/blob/master/merge/set_test.go)\n- [x] Apply works for items in associative lists with keys. [link](https://github.com/kubernetes-sigs/structured-merge-diff/blob/master/merge/key_test.go)\n- [x] Apply works for nested schemas, including recursive schemas. [link](https://github.com/kubernetes-sigs/structured-merge-diff/blob/master/merge/nested_test.go)\n- [x] Apply works for multiple appliers. [link](https://github.com/kubernetes-sigs/structured-merge-diff/blob/9f6585cadf64c6b61b5a75bde69ba07d5d34dc3f/merge/multiple_appliers_test.go#L31-L685)\n- [x] Apply works when the object conversion changes value of map keys. [link](https://github.com/kubernetes-sigs/structured-merge-diff/blob/9f6585cadf64c6b61b5a75bde69ba07d5d34dc3f/merge/multiple_appliers_test.go#L687-L886)\n- [x] Apply works when unknown/obsolete versions are present in managedFields (for when APIs are deprecated). [link](https://github.com/kubernetes-sigs/structured-merge-diff/blob/master/merge/obsolete_versions_test.go)\n\nUnit Tests for:\n\n- [x] Apply strips certain fields (like name and namespace) from managers. [link](https://github.com/kubernetes/kubernetes/blob/8a6a2883f9a38e09ae941b62c14f4e68037b2d21/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager_test.go#L69-L139)\n- [x] ManagedFields API can be round tripped through the structured-merge-diff format. [link](https://github.com/kubernetes/kubernetes/blob/4394bf779800710e67beae9bddde4bb5425ce039/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfields_test.go#L30-L156)\n- [x] Manager identifiers passed to structured-merge-diff are encoded as json. [link](https://github.com/kubernetes/kubernetes/blob/4394bf779800710e67beae9bddde4bb5425ce039/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfields_test.go#L158-L202)\n- [x] Managers will be sorted by operation, then timestamp, then manager name. [link](https://github.com/kubernetes/kubernetes/blob/4394bf779800710e67beae9bddde4bb5425ce039/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfields_test.go#L204-L304)\n- [x] Conflicts will be returned as readable status errors. [link](https://github.com/kubernetes/kubernetes/blob/69b9167dcbc8eea2ca5653fa42584539920a1fd4/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/conflict_test.go#L31-L106)\n- [x] Fields API can be round tripped through the structured-merge-diff format. [link](https://github.com/kubernetes/kubernetes/blob/0e1d50e70fdc9ed838d75a7a1abbe5fa607d22a1/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/fields_test.go#L29-L57)\n- [x] Fields API conversion to and from the structured-merge-diff format catches errors. [link](https://github.com/kubernetes/kubernetes/blob/0e1d50e70fdc9ed838d75a7a1abbe5fa607d22a1/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/fields_test.go#L59-L109)\n- [x] Path elements can be round tripped through the structured-merge-diff format. [link](https://github.com/kubernetes/kubernetes/blob/6b2e4682fe883eebcaf1c1e43cf2957dde441174/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/pathelement_test.go#L21-L54)\n- [x] Path element conversion will ignore unknown qualifiers. [link](https://github.com/kubernetes/kubernetes/blob/6b2e4682fe883eebcaf1c1e43cf2957dde441174/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/pathelement_test.go#L56-L61)\n- [x] Path element confersion will fail if a known qualifier's value is invalid. [link](https://github.com/kubernetes/kubernetes/blob/6b2e4682fe883eebcaf1c1e43cf2957dde441174/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/pathelement_test.go#L63-L84)\n- [x] Can convert both built-in objects and CRDs to structured-merge-diff typed objects. [link](https://github.com/kubernetes/kubernetes/blob/42aba643290c19a63168513bd758822e8014a0fd/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter_test.go#L40-L135)\n- [x] Can convert structured-merge-diff typed objects between API versions. [link](https://github.com/kubernetes/kubernetes/blob/0e1d50e70fdc9ed838d75a7a1abbe5fa607d22a1/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/versionconverter_test.go#L32-L69)\n\nIntegration tests for:\n\n- [x] Creating an object with apply works with default and custom storage implementations. [link](https://github.com/kubernetes/kubernetes/blob/1b8c8f1daf4b1ed6d17ee1d2f40d62c8ecec0e15/test/integration/apiserver/apply/apply_test.go#L55-L121)\n- [x] Create is blocked on apply if uid is provided. [link](https://github.com/kubernetes/kubernetes/blob/1b8c8f1daf4b1ed6d17ee1d2f40d62c8ecec0e15/test/integration/apiserver/apply/apply_test.go#L123-L154)\n- [x] Apply has conflicts when changing fields set by Update, and is able to force. [link](https://github.com/kubernetes/kubernetes/blob/1b8c8f1daf4b1ed6d17ee1d2f40d62c8ecec0e15/test/integration/apiserver/apply/apply_test.go#L156-L239)\n- [x] There are no changes to the managedFields API. [link](https://github.com/kubernetes/kubernetes/blob/1b8c8f1daf4b1ed6d17ee1d2f40d62c8ecec0e15/test/integration/apiserver/apply/apply_test.go#L241-L341)\n- [x] ManagedFields has no entries for managers who manage no fields. [link](https://github.com/kubernetes/kubernetes/blob/1b8c8f1daf4b1ed6d17ee1d2f40d62c8ecec0e15/test/integration/apiserver/apply/apply_test.go#L343-L392)\n- [x] Apply works with custom resources. [link](https://github.com/kubernetes/kubernetes/blob/b55417f429353e1109df8b3bfa2afc8dbd9f240b/staging/src/k8s.io/apiextensions-apiserver/test/integration/apply_test.go#L34-L117)\n- [x] Run kubectl apply tests with server-side flag enabled. [link](https://github.com/kubernetes/kubernetes/blob/81e6407393aa46f2695e71a015f93819f1df424c/test/cmd/apply.sh#L246-L314)\n\n## Graduation Criteria\n\nAn alpha version of this is targeted for 1.14.\n\nThis can be promoted to beta when it is a drop-in replacement for the existing\nkubectl apply, and has no regressions (which aren't bug fixes). This KEP will be\nupdated when we know the concrete things changing for beta.\n\nThis will be promoted to GA once it's gone a sufficient amount of time as beta\nwith no changes. A KEP update will precede this.\n\n## Implementation History\n\n* Early 2018: @lavalamp begins thinking about apply and writing design docs\n* 2018Q3: Design shift from merge + diff to tracking field managers.\n* 2019Q1: Alpha.\n\n(For more details, one can view the apply-wg recordings, or join the mailing list\nand view the meeting notes. TODO: links)\n\n## Drawbacks\n\nWhy should this KEP _not_ be implemented: many bugs in kubectl apply will go\naway. Users might be depending on the bugs.\n\n## Alternatives\n\nIt's our belief that all routes to fixing the user pain involve\ncentralizing this functionality in the control plane.\n"
+  },
+  {
+    "id": "750a2db91d362a99c5706ad20d4bd5d7",
+    "title": "Dry-run",
+    "authors": ["@apelisse"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["sig-api-machinery", "sig-cli"],
+    "reviewers": ["@lavalamp", "@deads2k"],
+    "approvers": ["@erictune"],
+    "editor": "apelisse",
+    "creationDate": "2018-06-21",
+    "lastUpdated": "2018-06-21",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Kubernetes Dry-run\n\nDry-run is a new feature that we intend to implement in the api-server. The goal\nis to be able to send requests to modifying endpoints, and see if the request\nwould have succeeded (admission chain, validation, merge conflicts, ...) and/or\nwhat would have happened without having it actually happen. The response body\nfor the request should be as close as possible to a non dry-run response.\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Specifying dry-run](#specifying-dry-run)\n- [Admission controllers](#admission-controllers)\n- [Generated values](#generated-values)\n- [Storage](#storage)\n- [kubectl](#kubectl)\n\u003c!-- /toc --\u003e\n\n## Specifying dry-run\n\nDry-run is triggered by setting the “dryRun” query parameter on modifying\nverbs: POST, PUT, PATCH and DELETE.\n\nThis parameter is a string, working as an enum:\n- All: Everything will run as normal, except for the storage that won’t be\n  modified. Everything else should work as expected: admission controllers will\n  be run to check that the request is valid, mutating controllers will change\n  the object, merge will be performed on PATCH. The storage layer will be\n  informed not to save, and the final object will be returned to the user with\n  normal status code.\n- Leave the value empty, or don't specify the parameter at all to keep the\n  default modifying behavior.\n\nNo other values are supported yet, but this gives us an opportunity to create a\nfiner-grained mechanism later, if necessary.\n\n## Admission controllers\n\nAdmission controllers need to be modified to understand that the request is a\n“dry-run” request. Admission controllers are allowed to have side-effects\nwhen triggered, as long as there is a reconciliation system, because it is not\nguaranteed that subsequent validating will permit the request to finish.\nQuotas for example uses the current request values to change the available quotas.\nThe ```admission.Attributes``` interface will be edited like this, to inform the\nbuilt-in admission controllers if a request is a dry-run:\n```golang\ntype Attributes interface {\n\t...\n\t// IsDryRun indicates that modifications will definitely not be persisted for this request. This is to prevent\n\t// admission controllers with side effects and a method of reconciliation from being overwhelmed.\n\t// However, a value of false for this does not mean that the modification will be persisted, because it\n\t// could still be rejected by a subsequent validation step.\n\tIsDryRun() bool\n\t...\n}\n```\n\nAll built-in admission controllers will then have to be checked, and the ones with side\neffects will have to be changed to handle the dry-run case correctly. Some examples of\nbuilt-in admission controllers with the possibility for side-effects are:\n- ResourceQuota\n- EventRateLimit\n- NamespaceAutoProvision\n- (Valid|Mut)atingAdmissionWebhook\n\nTo address the possibility of webhook authors [relying on side effects](https://github.com/kubernetes/website/blame/836629cb118e0f74545cc7d6d97aa6b9edfa1a16/content/en/docs/reference/access-authn-authz/admission-controllers.md#L582-L584), a new field\nwill be added to ```admissionregistration.k8s.io/v1beta1.ValidatingWebhookConfiguration``` and\n```admissionregistration.k8s.io/v1beta1.MutatingWebhookConfiguration``` so that webhooks\ncan explicitly register as having dry-run support.\nIf dry-run is requested on a non-supported webhook, the request will be completely rejected,\nas a 400: Bad Request. This field will be defaulted to true and deprecated in v1, and completely removed in v2.\nAll webhooks registered with v2 will be assumed to support dry run. The [api conventions](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md) advise\nagainst bool fields because \"many ideas start as boolean but eventually trend towards a small set\nof mutually exclusive options\" but in this case, we plan to remove the field in a future version.\n```golang\n// admissionregistration.k8s.io/v1beta1\n...\ntype Webhook struct {\n\t...\n\t// DryRunnable defines whether this webhook will correctly handle dryRun requests.\n\t// If false, any dryRun requests to resources/subresources this webhook applies to\n\t// will be completely rejected and the webhook will not be called.\n\t// Defaults to false.\n\t// +optional\n\tDryRunnable *bool `json:\"dryRunnable,omitempty\" protobuf:\"varint,6,number,opt,name=dryRunnable\"`\n}\n```\n\nAdditionally, a new field will be added to ```admission.k8s.io/v1beta1.AdmissionReview```\nAPI object to reflect the changes to the ```admission.Attributes``` interface, indicating\nwhether or not the request being reviewed is for a dry-run:\n```golang\n// admission.k8s.io/v1beta1\n...\ntype AdmissionRequest struct {\n\t...\n\t// DryRun indicates that modifications will definitely not be persisted for this request.\n\t// Defaults to false.\n\t// +optional\n\tDryRun *bool `json:\"dryRun,omitempty\" protobuf:\"varint,11,number,opt,name=dryRun\"`\n}\n```\n\n## Generated values\n\nSome values of the object are typically generated before the object is persisted:\n- generateName can be used to assign a unique random name to the object,\n- creationTimestamp/deletionTimestamp records the time of creation/deletion,\n- UID uniquely identifies the object and is randomly generated (non-deterministic),\n- resourceVersion tracks the persisted version of the object.\n\nMost of these values are not useful in the context of dry-run, and could create\nsome confusion. The UID and the generated name would have a different value in a\ndry-run and non-dry-run creation. These values will be left empty when\nperforming a dry-run.\n\nCreationTimestamp and DeletionTimestamp are also generated on creation/deletion,\nbut there are less ways to abuse them so they will be generated as they for a\nregular request.\n\nResourceVersion will also be left empty on creation. On updates, the value will\nstay unchanged.\n\n## Storage\n\nThe storage layer will be modified, so that it can know if request is dry-run,\nmost likely by looking for the field in the “Options” structure (missing for\nsome handlers, to be added). If it is, it will NOT store the object, but return\nsuccess. That success can be forwarded back to the user.\n\nA dry-run request should behave as close as possible to a regular\nrequest. Attempting to dry-run create an existing object will result in an\n`AlreadyExists` error to be returned. Similarly, if a dry-run update is\nperformed on a non-existing object, a `NotFound` error will be returned.\n\n## kubectl\n\nFor the `kubectl` client integration with server-side dry-run, we will pass\nthe `dryRun` query parameter by reading the user's intent from a flag.\n\nFor beta, we use `--server-dry-run` for `kubectl apply` to exercise\nserver-side apply. This flag will be deprecated next release, then removed after 1 release.\n\nFor GA, we'll use the existing `--dry-run` flag available on `kubectl apply`.\n\nCurrently, the `--dry-run` flag is a boolean for subcommands including\n`kubectl apply` and `kubectl create`.\n\nWe'll extend the flag to accept strings for new options\nfor selecting client-side and server-side behavior: `none`, `client` and `server`.\n\nFor backwards compatibility, we'll continue to default the value for `--dry-run`\nto `--dry-run=client`, which is equivalent to the existing behavior for\n`--dry-run=true`.\n\nThe boolean values for `--dry-run` will be deprecated next release, then removed in 2 releases.\n\nThe default value for `--dry-run` with the flag set and unspecified\nwill be deprecated in the next release, then in 2 releases we will\nrequire that a value must be specified.\nUsers must update any scripts to explicitly set `--dry-run=client` or\n`--dry-run=server`.\n\nThe `--dry-run` flag will be parsed and defaulted as described below.\n\n```golang\nfunc ParseDryRun(dryRunFlag string) (DryRunEnum, error) {\n  if dryRunFlag == \"\" {\n    klog.Warning(`The unset value for --dry-run is deprecated and a value will be required in a future version. Must be \"none\", \"server\", or \"client\".`)\n    // this warning will eventually become a fatal error\n  }\n  b, err := strconv.ParseBool(dryRunFlag)\n  if err != nil { // The flag is not a boolean\n    switch dryRunFlag {\n    case \"client\":\n      return DryRunClient, nil\n    case \"server\":\n      return DryRunServer, nil\n    case \"none\":\n      return DryRunNone, nil\n    default:\n      return DryRunNone, fmt.Errorf(`Invalid dry-run value (%v). Must be \"none\", \"server\", or \"client\".`, dryRunFlag)\n    }\n  }\n  if b { // flag was a boolean, and indicates true, run client-side\n    klog.Warning(`Boolean values for --dry-run are deprecated and will be removed in a future version. Must be \"none\", \"server\", or \"client\".`)\n    return DryRunClient, nil\n  }\n  return DryRunNone, nil\n}\n\nfunc AddDryRunFlag(cmd *cobra.Command) {\n\tcmd.Flags().String(\n      \"dry-run\",\n      \"client\",\n      `Must be \"none\", \"server\", or \"client\". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`\n      )\n}\n```\n\n"
+  },
+  {
+    "id": "4aab45b120e747797f76bd1bc412c71f",
+    "title": "Migrating API objects to latest storage version",
+    "authors": ["@xuchao"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": null,
+    "reviewers": ["@deads2k", "@yliaog", "@lavalamp"],
+    "approvers": ["@deads2k", "@lavalamp"],
+    "editor": "",
+    "creationDate": "2018-08-06",
+    "lastUpdated": "2019-03-19",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Migrating API objects to latest storage version\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n- [Proposal](#proposal)\n  - [Alpha workflow](#alpha-workflow)\n  - [Alpha API](#alpha-api)\n  - [Failure recovery](#failure-recovery)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Beta Graduation Criteria](#beta-graduation-criteria)\n- [Alternatives](#alternatives)\n  - [update-storage-objects.sh](#update-storage-objectssh)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nWe propose a solution to migrate the stored API objects in Kubernetes clusters.\nIn 2018 Q4, we will deliver a tool of alpha quality. The tool extends and\nimproves based on the [oc adm migrate storage][] command.\n\n[oc adm migrate storage]:https://www.mankier.com/1/oc-adm-migrate-storage\n\n## Motivation\n\n\"Today it is possible to create API objects (e.g., HPAs) in one version of\nKubernetes, go through multiple upgrade cycles without touching those objects,\nand eventually arrive at a version of Kubernetes that can’t interpret the stored\nresource and crashes. See k8s.io/pr/52185.\"[1][]. We propose a solution to the\nproblem.\n\n[1]:https://docs.google.com/document/d/1eoS1K40HLMl4zUyw5pnC05dEF3mzFLp5TPEEt4PFvsM\n\n### Goals\n\nA successful storage version migration tool must:\n* work for Kubernetes built-in APIs, custom resources (CR), and aggregated APIs.\n* do not add burden to cluster administrators or Kubernetes distributions.\n* only cause insignificant load to apiservers. For example, if the master has\n  10GB memory, the migration tool should generate less than 10 qps of single\n  object operations(TODO: measure the memory consumption of PUT operations;\n  study how well the default 10 Mbps bandwidth limit in the oc command work).\n* work for big clusters that have ~10^6 instances of some resource types.\n* make progress in flaky environment, e.g., flaky apiservers, or the migration\n  process get preempted.\n* allow system administrators to track the migration progress.\n\nWe will deliver a vendor-agnostic solution to automatically detect and migrate\nresources when the default storage version has changed.\n\n## Proposal\n\n### Alpha workflow\n\nAt the alpha stage, the migrator needs to be manually launched, and does not\nhandle custom resources or aggregated resources.\n\nAfter all the kube-apiservers are at the desired version, the cluster\nadministrator runs `kubectl apply -f migrator-initializer-\u003ck8s-versio\u003e.yaml`.\nThe apply command\n* creates a *kube-storage-migration* namespace\n* creates a *storage-migrator* service account\n* creates a *system:storage-migrator* cluster role that can *get*, *list*, and\n  *update* all resources, and in addition, *create* and *delete* CRDs.\n* creates a cluster role binding to bind the created service account with the\n  cluster role\n* creates a **migrator-initializer** job running with the\n  *storage-migrator* service account.\n\nThe **migrator-initializer** job\n* deletes any existing deployment of **kube-migrator controller**\n* creates a **kube-migrator controller** deployment running with the\n  *storage-migrator* service account.\n* generates a comprehensive list of resource types via the discovery API\n* discovers all custom resources via listing CRDs\n* discovers all aggregated resources via listing all `apiservices` that have\n  `.spec.service != null`\n* removes the custom resources and aggregated resources from the comprehensive\n  resource list. The list now only contains Kubernetes built-in resources.\n* removes resources that share the same storage. At the alpha stage, the\n  information is hard-coded, like in this [list][].\n* creates `migration` CRD (see the [API section][] for the schema) if it does\n  not exist.\n* creates `migration` CRs for all remaining resources in the list. The\n  `ownerReferences` of the `migration` objects are set to the **kube-migrator\n  controller** deployment. Thus, the old `migration`s are deleted with the old\n  deployment in the first step.\n\nThe control loop of **kube-migrator controller** does the following:\n* runs a reflector to watch for the instances of the `migration` CR. The list\n  function used to construct the reflector sorts the `migration`s so that the\n  *Running* `migration` will be processed first.\n* syncs one `migration` at a time to avoid overloading the apiserver,\n  * if `migration.status` is nil, or `migration.status.conditions` shows\n    *Running*, it creates a **migration worker** goroutine to migrate the\n    resource type.\n  * adds the *Running* condition to `migration.status.conditions`.\n  * waits until the **migration worker** goroutine finishes, adds either the\n    *Succeeded* or *Failed* condition to `migration.status.conditions` and sets\n    the *Running* condition to false.\n\nThe **migration worker** runs the equivalence of `oc adm migrate storage\n--include=\u003cresource type\u003e` to migrate a resource type. The **migration worker**\nuses API chunking to retrieve partial lists of a resource type and thus can\nmigrate a small chunk at a time. It stores the [continue token] in the owner\n`migration.spec.continueToken`. With the inconsistent continue token\nintroduced in [#67284][], the **migration worker** does not need to worry about\nexpired continue token.\n\n[list]:https://github.com/openshift/origin/blob/2a8633598ef0dcfa4589d1e9e944447373ac00d7/pkg/oc/cli/admin/migrate/storage/storage.go#L120-L184\n[#67284]:https://github.com/kubernetes/kubernetes/pull/67284\n[API section]:#alpha-api\n\nThe cluster admin can run the `kubectl wait --for=condition=Succeeded\nmigrations` to wait for all migrations to succeed.\n\nUsers can run `kubectl create` to create `migration`s to request migrating\ncustom resources and aggregated resources.\n\n### Alpha API\n\nWe introduce the `storageVersionMigration` API to record the intention and the\nprogress of a migration. Throughout this doc, we abbreviated it as `migration`\nfor simplicity. The API will be a CRD defined in the `migration.k8s.io` group.\n\nRead the [workflow section][] to understand how the API is used.\n\n```golang\ntype StorageVersionMigration struct {\n  metav1.TypeMeta\n  // For readers of this KEP, metadata.generateName will be \"\u003cresource\u003e.\u003cgroup\u003e\"\n  // of the resource being migrated.\n  metav1.ObjectMeta\n  Spec StorageVersionMigrationSpec\n  Status StorageVersionMigrationStatus\n}\n\n// Note that the spec only contains an immutable field in the alpha version. To\n// request another round of migration for the resource, clients need to create\n// another `migration` CR.\ntype StorageVersionMigrationSpec {\n  // Resource is the resource that is being migrated. The migrator sends\n  // requests to the endpoint tied to the Resource.\n  // Immutable.\n  Resource GroupVersionResource\n  // ContinueToken is the token to use in the list options to get the next chunk\n  // of objects to migrate. When the .status.conditions indicates the\n  // migration is  \"Running\", users can use this token to check the progress of\n  // the migration.\n  // +optional\n  ContinueToken string\n}\n\ntype MigrationConditionType string\n\nconst (\n  // MigrationRunning indicates that a migrator job is running.\n  MigrationRunning MigrationConditionType = \"Running\"\n  // MigrationSucceed indicates that the migration has completed successfully.\n  MigrationSucceeded MigrationConditionType = \"Succeeded\"\n  // MigrationFailed indicates that the migration has failed.\n  MigrationFailed MigrationConditionType = \"Failed\"\n)\n\ntype MigrationCondition struct {\n\t// Type of the condition\n\tType MigrationConditionType\n\t// Status of the condition, one of True, False, Unknown.\n\tStatus corev1.ConditionStatus\n\t// The last time this condition was updated.\n\tLastUpdateTime metav1.Time\n\t// The reason for the condition's last transition.\n\tReason string\n\t// A human readable message indicating details about the transition.\n\tMessage string\n}\n\ntype StorageVersionMigrationStatus {\n  // Conditions represents the latest available observations of the migration's\n  // current state.\n  Conditions []MigrationCondition\n}\n```\n\n[continue token]:https://github.com/kubernetes/kubernetes/blob/972e1549776955456d9808b619d136ee95ebb388/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go#L82\n[workflow section]:#alpha-workflow\n\n### Failure recovery\n\nAs stated in the goals section, the migration has to make progress even if the\nenvironment is flaky. This section describes how the migrator recovers from\nfailure.\n\nKubernetes **replicaset controller** restarts the **migration controller** `pod`\nif it fails. Because the migration states, including the continue token, are\n  stored in the `migration` object, the **migration controller** can resume from\n  where it left off.\n\n[workflow section]:#alpha-workflow\n\n### Beta workflow - Automation\n\nIt is a beta goal to automate the migration workflow. That is, migration does\nnot need to be triggered manually by cluster admins, or by custom control loops\nof Kubernetes distributions.\n\nThe [storage version hash][] is added to the Kubernetes discovery document as an\nalpha feature in 1.14. A [triggering controller][] is added to poll the discovery\ndocument, and creates migrations when the storage version hash of a resource\nchanges. See [KEP][] for the details on the automated migration workflow.\n\n[storage version hash]:https://github.com/kubernetes/kubernetes/pull/73191\n[triggering controller]:https://github.com/kubernetes-sigs/kube-storage-version-migrator/pull/21\n[KEP]:storage-migration-auto-trigger.md\n\n### Risks and Mitigations\n\nThe migration process does not change the objects, so it will not pollute\nexisting data.\n\nIf the rate limiting is not tuned well, the migration can overload the\napiserver. Users can delete the migration controller and the migration\njobs to mitigate.\n\nBefore upgrading or downgrading the cluster, the cluster administrator must run\n`kubectl wait --for=condition=Succeeded migrations` to make sure all\nmigrations have completed. Otherwise the apiserver can crash, because it cannot\ninterpret the serialized data in etcd. To mitigate, the cluster administrator\ncan rollback the apiserver to the old version, and wait for the migration to\ncomplete. \n\nWith the newly introduced [storageStates API][], the cluster administrator can\nfast upgrade/downgrade as long as the new apiserver binaries understand all\nversions recorded in storageState.status.persistedStorageVersionHashes.\n\n[storageStates API]:storage-migration-auto-trigger.md#api-design\n\n## Beta Graduation Criteria\n\n* Visibility\n  * metrics for the number of migrated objects per resource. This metric also\n    indirectly manifests the speed migration per resource.\n  * metrics for pending, succeeded, and failed migrations. This metric also\n    indirectly manifests the frequency of migrations per resource.\n\n* End-to-end testing\n  * testing migration for CRDs\n  * chaos testing the migrator with injected network errors, and injected\n    crashes of both the migrator and the apiserver.\n  * stress testing the migrator with large lists of to-be-migrated objects\n  * optional: integrating the migrator into the Kubernetes upgrade tests,\n    verifying all objects are readable after the cluster upgrade\n\n* Deployment\n  * example manifests for installation (manifests are currently kept at\n    https://github.com/kubernetes-sigs/kube-storage-version-migrator/tree/master/manifests).\n\n## Alternatives\n\n### update-storage-objects.sh\n\nThe Kubernetes repo has an update-storage-objects.sh script. It is not\nproduction ready: no rate limiting, hard-coded resource types, no persisted\nmigration states. We will delete it, leaving a breadcrumb for any users to\nfollow to the new tool.\n"
+  },
+  {
+    "id": "39eb52f90ef0013602b3c624332214e1",
+    "title": "Graduate Admission Webhooks to GA",
+    "authors": ["@mbohlool"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": null,
+    "reviewers": ["@liggitt", "@deads2k"],
+    "approvers": ["@liggitt", "@deads2k"],
+    "editor": "TBD",
+    "creationDate": "2019-01-27",
+    "lastUpdated": "2019-02-04",
+    "status": "implemented",
+    "seeAlso": [
+      "[Admission Control Webhook Beta Design Doc](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/admission-control-webhooks.md)"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Admission Webhooks to GA\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Object selector](#object-selector)\n  - [Scope](#scope)\n  - [timeout configuration](#timeout-configuration)\n  - [Port configuration](#port-configuration)\n  - [Plumbing existing objects to delete admission](#plumbing-existing-objects-to-delete-admission)\n  - [Mutating Plugin ordering](#mutating-plugin-ordering)\n  - [Passing {Operation}Option to Webhook](#passing-operationoption-to-webhook)\n    - [Connection Options](#connection-options)\n  - [AdmissionReview v1](#admissionreview-v1)\n  - [Convert to webhook-requested version](#convert-to-webhook-requested-version)\n- [V1 API](#v1-api)\n- [V1beta1 changes](#v1beta1-changes)\n- [Validations](#validations)\n- [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Post-GA tasks](#post-ga-tasks)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nAdmission webhook is a way to extend kubernetes by putting hook on object creation/modification/deletion.\nAdmission webhooks can mutate or validate the object. This feature has been Beta since Kubernetes 1.9.\nThis document outline required steps to graduate it to GA.\n\n## Motivation\n\nAdmission webhooks are currently widely used for extending kubernetes but has been in beta for 3 releases.\nCurrent set of feature requests and bug reports on the feature shows it is close to be stable and motivation\nof this KEP is to address those feature requests and bug reports to move this feature toward general availability (GA).\n\n### Goals\n\nBased on the user feedback, These are the planned changes to current feature to graduate it to GA:\n\n* Object Selector: Add a general selector for objects that a webhook can target.\n* Scope: Define the scope of the targeted objects, e.g. cluster vs. namespaced.\n* timeout configuration: Add a configuration field to shorten the timeout of the webhook call.\n* port configuration: Add a configuration field to change the service port from the default 443.\n* AdmissionReview version selection: Add a API field to set the AdmissionReview versions a webhook can understand.\n* Pass existing objects to delete admission webhooks\n* re-run mutating plugins if any webhook changed object to fix the plugin ordering problem\n* pass OperationOption (such as CreateOption/DeleteOption) to the webhook\n* make `Webhook.SideEffects` field required in `v1` API (look at [dryRun KEP(https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/0015-dry-run.md#admission-controllers)] for more information on this item)\n* convert incoming objects to the webhook-requested group/version\n\n### Non-Goals\n\nThese are the issues considered and rejected for GA:\n\n* Can CA trust bundle being referenced via secret/configmap instead of inline? ([discussion issue](https://github.com/kubernetes/kubernetes/issues/72944))\n* Should API servers have a dynamic way to authenticate to admission webhooks? ([Discussion issue](https://github.com/kubernetes/enhancements/pull/658))\n\n## Proposal\n\nThis section describes each of the goals in the [Goals](#goals) section in more detail.\n\n### Object selector\n\nCurrently Admission Webhook supports namespace selectors, but that may not be enough \nfor some cases that admission webhook need to be skipped on some objects. For example \nif the Admission Webhook is running inside cluster and its rules includes wildcards \nwhich match required API objects for its own execution, it won't work. An object selector \nwould be useful exclude those objects. \n\nAlso in case of an optional webhooks, an object selector gives the end user ability to \ninclude or exclude an object without having access to admission webhook configuration \nwhich is normally restricted to cluster admins.\n\nNote that namespaced objects must match both the object selector (if specified) and namespace selector to be sent to the Webhook.\n\nThe object selector applies to the labels of the Object and OldObject sent to the webhook.\n\nAn empty object selector matches all requests and objects.\n\nA non-empty object selector is evaluated against the labels of both the old and new\nobjects that would be sent to the webhook. If the selector matches against either\nset of labels, the selector is considered to match the request.\n\nIf one of the objects is nil (like `OldObject` is for create requests, or `Object` is for delete requests),\nor cannot have labels (like a `DeploymentRollback` or `NodeProxyOptions` object), that object is not considered to match.\n\nThe proposed change is to add an ObjectSelector to the webhook API both in v1 and v1beta1:\n\n```golang\ntype {Validating,Mutating}Webhook struct {\n    ...\n    // ObjectSelector decides whether to run the webhook based on if the\n    // object has matching labels. objectSelector is evaluated against both\n    // the oldObject and newObject that would be sent to the webhook, and\n    // is considered to match if either object matches the selector. A null\n    // object (oldObject in the case of create, or newObject in the case of\n    // delete) or an object that cannot have labels (like a\n    // DeploymentRollback or a PodProxyOptions object) is not considered to\n    // match.\n    // Use the object selector only if the webhook is opt-in, because end\n    // users may skip the admission webhook by setting the labels.\n    // Default to the empty LabelSelector, which matches everything.\n    // +optional\n    ObjectSelector *metav1.LabelSelector `json:\"objectSelector,omitempty\"\n}\n```\n\n### Scope\n\nCurrent webhook Rules applies to objects of all scopes. That means a Rule can use wildcards \nto target both namespaced and cluster-scoped objects. \n\nAn evaluation of the targeting capabilities required by in-tree admission plugins showed that\nsome plugins (like NamespaceLifecycle and ResourceQuota) require the ability to intercept\nall namespaced resources. This selection is currently inexpressible for webhook admission.\n\nThe proposal is to add a scope field to Admission Webhook configuration to limit webhook \ntargeting to namespaced or cluster-scoped objects. This enables webhook developers to \ntarget only namespaced objects or cluster-scoped objects, just like in-tree admission plugins can.\n\nThe field will be added to both v1 and v1beta1.\n\nThe field is optional and defaults to \"*\", meaning no scope restriction.\n\n```golang\ntype ScopeType string\n\nconst (\n     // ClusterScope means that scope is limited to cluster-scoped objects.\n     // Namespace objects are cluster-scoped.\n     ClusterScope ScopeType = \"Cluster\"\n     // NamespacedScope means that scope is limited to namespaced objects.\n     NamespacedScope ScopeType = \"Namespaced\"\n     // AllScopes means that all scopes are included.\n     AllScopes ScopeType = \"*\"\n)\n\ntype Rule struct {\n    ...\n\n     // Scope specifies the scope of this rule.\n     // Valid values are \"Cluster\", \"Namespaced\", and \"*\"\n     // \"Cluster\" means that only cluster-scoped resources will match this rule.\n     // Namespace API objects are cluster-scoped.\n     // \"Namespaced\" means that only namespaced resources will match this rule.\n     // \"*\" means that there are no scope restrictions.\n     // Default is \"*\".\n     //\n     // +optional\n     Scope ScopeType `json:\"scope,omitempty\" protobuf:\"bytes,3,opt,name=scope\"`\n}\n```\n\n### timeout configuration\n\nAdmission Webhook has a default timeout of 30 seconds today, but long running webhooks \nwould result in a poor performance. By adding a timeout field to the configuration, \nthe webhook author can limit the running time of the webhook that either result in \nfailing the API call earlier or ignore the webhook call based on the failure policy. \nThis feature, however, would not let the timeout to be extended more than 30 seconds \nand the timeout would be defaulted to a shorter value (e.g. 10 seconds) for v1 API while \nstays 30 second for v1beta API to keep backward compatibility.\n\n```golang\ntype {Validating,Mutating}Webhook struct {\n    ...\n     // TimeoutSeconds specifies the timeout for this webhook. After the timeout passes,\n     // the webhook call will be ignored or the API call will fail based on the\n     // failure policy.\n     // The timeout value must be between 1 and 30 seconds.\n     // Default to 10 seconds for v1 API and 30 seconds for v1beta1 API.\n     // +optional\n     TimeoutSeconds *int32 `json:\"timeoutSeconds,omitempty\" protobuf:\"varint,8,opt,name=timeoutSeconds\"`\n}\n```\n\n### Port configuration\n\nToday Admission Webhook port is always expected to be 443 on service reference. But this \nlimitation was arbitrary and there are cases that Admission Webhook cannot use this port. \nThis feature will add a port field to service based webhooks and allows specifying a port \nother than 443 for service based webhooks. Specifying port should already be available for \nURL based webhooks.\n\nThe following field will be added to the `ServiceReference` types used by admission webhooks, `APIService`, and `AuditSink` configurations:\n\n```golang\ntype ServiceReference struct {\n    ...\n\n     // If specified, the port on the service that hosting webhook.\n     // Default to 443 for backward compatibility.\n     // `Port` should be a valid port number (1-65535, inclusive).\n     // +optional\n     Port *int32 `json:\"port,omitempty\" protobuf:\"varint,4,opt,name=port\"`\n}\n```\n\n### Plumbing existing objects to delete admission\n\nCurrent admission webhooks can hook on `DELETE` events but they won't get any object in \n`oldObject` or `Object` field of `AdmissionRequest`. The proposal is to send them existing \nobject(s) as the `oldObject`. This is also helpful for applying `ObjectSelector` to these \nwebhooks. When deleting a collection, each matching object is sent as the `oldObject` to\nto webhooks in individual request.\n\nThere is no API change for this feature, only documentation:\n\n```golang\ntype AdmissionRequest struct {\n  ...\n  // OldObject is the existing object. Only populated for UPDATE and DELETE requests.\n  // +optional\n  OldObject runtime.RawExtension `json:\"oldObject,omitempty\" protobuf:\"bytes,10,opt,name=oldObject\"`\n  ..\n}\n```\n\n### Mutating Plugin ordering\n\nA single ordering of mutating admissions plugins (including webhooks) does not work for all cases\n(see https://issue.k8s.io/64333 as an example). A mutating webhook can add a new sub-structure \nto the object (like adding a `container` to a pod), and other mutating plugins which have already \nrun may have opinions on those new structures (like setting an `imagePullPolicy` on all containers). \n\nTo allow mutating admission plugins to observe changes made by other plugins, regardless of order,\nadmission will add the ability to re-run mutating plugins (including webhooks) a single time if \nthere is any change made by a mutating webhook on the first pass.\n\n1. Initial mutating admission pass\n2. If any webhook plugins returned patches modifying the object, do a second mutating admission pass\n   1. Run all in-tree mutating admission plugins\n   2. Run all applicable webhook mutating admission plugins that indicate they want to be reinvoked\n\nMutating plugins that are reinvoked must be idempotent, able to successfully process an object they have already \nadmitted and potentially modified. This will be clearly documented in admission webhook documentation,\nexamples, and test guidelines. Mutating admission webhooks *should* already support this (since any \nchange they can make in an object could already exist in the user-provided object), and any webhooks\nthat do not are broken for some set of user input.\n\nNote that idempotence (whether a webhook can successfully operate on its own output) is distinct from\nthe `sideEffects` indicator, which describes whether a webhook can safely be called in `dryRun` mode.\n\nThe reinvocation behavior will be opt-in, so `reinvocationPolicy` defaults to `\"Never\"`.\n\n```golang\ntype MutatingWebhook struct {\n    ...\n    // reinvocationPolicy indicates whether this webhook should be called multiple times as part of a single admission evaluation.\n    // Allowed values are \"Never\" and \"IfNeeded\".\n    //\n    // Never: the webhook will not be called more than once in a single admission evaluation.\n    //\n    // IfNeeded: the webhook will be called at least one additional time as part of the admission evaluation\n    // if the object being admitted is modified by other admission plugins after the initial webhook call.\n    // Webhooks that specify this option *must* be idempotent, able to process objects they previously admitted.\n    // Note:\n    // * the number of additional invocations is not guaranteed to be exactly one.\n    // * if additional invocations result in further modifications to the object, webhooks are not guaranteed to be invoked again.\n    // * webhooks that use this option may be reordered to minimize the number of additional invocations.\n    // * to validate an object after all mutations are guaranteed complete, use a validating admission webhook instead (recommended for webhooks with side-effects).\n    //\n    // Defaults to \"Never\".\n    // +optional\n    ReinvocationPolicy *ReinvocationPolicyType `json:\"reinvocationPolicy,omitempty\"`\n}\n\ntype ReinvocationPolicyType string\n\nvar (\n  NeverReinvocationPolicy ReinvocationPolicyType = \"Never\"\n  IfNeededReinvocationPolicy ReinvocationPolicyType = \"IfNeeded\"\n)\n```\n\nAlthough this problem can go deeper than two levels (for example, if the second mutating webhook added\na new structure based on the new `container`), a single re-run is sufficient for the current set of \nin-tree plugins and webhook use-cases. If future use cases require additional or more targeted reinvocations,\nthe `AdmissionReview` response can be enhanced to provide information to target those reinvocations.\n\nTest scenarios:\n\n1. No reinvocation\n   * in-tree -\u003e mutation\n   * webhook -\u003e no mutation\n   * complete (no reinvocation needed)\n\n2. In-tree reinvocation only\n   * in-tree -\u003e mutation\n   * webhook -\u003e mutation\n   * reinvoke in-tree -\u003e no mutation\n   * complete (in-tree didn't change anything, so no webhook reinvocation required)\n\n3. Full reinvocation\n   * in-tree -\u003e mutation\n   * webhook -\u003e mutation\n   * reinvoke in-tree -\u003e mutation\n   * reinvoke webhook -\u003e mutation\n   * complete (both reinvoked once, no additional loops)\n\n4. Multiple webhooks, partial reinvocation\n   * in-tree -\u003e mutation\n   * webhook A -\u003e mutation\n   * webhook B -\u003e mutation\n   * reinvoke in-tree -\u003e no mutation\n   * reinvoke webhook A -\u003e no mutation\n   * complete (no mutations occurred after webhook B was called, no reinvocation required)\n\n5. Multiple webhooks, full reinvocation\n   * in-tree -\u003e mutation\n   * webhook A -\u003e mutation\n   * webhook B -\u003e mutation\n   * reinvoke in-tree -\u003e no mutation\n   * reinvoke webhook A -\u003e mutation\n   * reinvoke webhook B -\u003e mutation (webhook A mutated after webhook B was called, one reinvocation required)\n   * complete (both reinvoked once, no additional loops)\n\n### Passing {Operation}Option to Webhook\n\nEach of the operations webhook can have an `Option` structure (e.g. `DeleteOption` or `CreateOption`) \npassed by the user. It is useful for some webhooks to know what are those options user requested to \nbetter modify or validate object. The proposal is to add those Options as an `Options` field to the \n`AdmissionRequest` API object.\n\n```golang\ntype AdmissionRequest struct {\n  ...\n  // Options is the operation option structure user passed to the API call. e.g. `DeleteOption` or `CreateOption`.\n  // +optional\n  Options runtime.RawExtension `json:\"options,omitempty\" protobuf:\"bytes,12,opt,name=options\"`\n  ..\n}\n```\n\n#### Connection Options\n\nEach connect operation declares its own options type, which is sent as the\n`AdmissionRequest.Object` to admission webhooks. For example, the published\nOpenAPI for node proxy declares its connect operation as:\n\n```json\n\"/api/v1/nodes/{name}/proxy\": {\n  ...\n  \"x-kubernetes-action\": \"connect\",\n  \"x-kubernetes-group-version-kind\": {\n    \"group\": \"\",\n    \"kind\": \"NodeProxyOptions\",\n    \"version\": \"v1\"\n  }\n}\n```\n\nHere, an admission webhook request for a node proxy connect operation will send\n`NodeProxyOptions` as the `AdmissionRequest.Object` and since connect operations\nhave no common options, `AdmissionRequest.Options` will be absent.\n\nThis is consistent with how kubernetes operations send the type specified by the\nOpenAPI `x-kubernetes-group-version-kind` property as the\n`AdmissionRequest.Object` to admission webhooks.\n\n### AdmissionReview v1\n\nThe payload API server sends to Admission webhooks is called AdmissionReview which is `v1beta1` today.\nThe proposal is to promote the API to v1 with no change to the API object definition. Because of different \nversions of Admission Webhooks, there must be a way for the webhook developer to specify which apiVersion \nof `AdmissionReview` they support. The field must be an ordered list which reflects the webhooks preference \nof `AdmissionReview` apiVersions.\n\nA version list will be added to webhook configuration that would be defaulted to `['v1beta1']` in v1beta1 API \nand will be a required field in `v1`.\n\nA webhook must respond with the same apiVersion of the AdmissionReview it received. \nFor example, a webhook that registers admissionReviewVersions:[\"v1\",\"v1beta1\"] must be prepared to receive and respond with those versions. \n\nV1 API looks like this:\n\n```golang\ntype {Validating,Mutating}Webhook struct {\n    ...\n\n     // AdmissionReviewVersions is an ordered list of preferred `AdmissionReview`\n     // versions the Webhook expects. API server will try to use first version in\n     // the list which it supports. If none of the versions specified in this list\n     // supported by API server, validation will fail for this object.\n     // If the webhook configuration has already been persisted, calls to the \n     // webhook will fail and be subject to the failure policy.\n     // This field is required and cannot be empty.\n     AdmissionReviewVersions []string `json:\"admissionReviewVersions\" protobuf:\"bytes,9,rep,name=admissionReviewVersions\"`\n}\n```\n\nV1beta1 API looks like this:\n\n```golang\ntype {Validating,Mutating}Webhook struct {\n    ...\n\n     // AdmissionReviewVersions is an ordered list of preferred `AdmissionReview`\n     // versions the Webhook expects. API server will try to use first version in\n     // the list which it supports. If none of the versions specified in this list\n     // supported by API server, validation will fail for this object.\n     // If the webhook configuration has already been persisted, calls to the \n     // webhook will fail and be subject to the failure policy.\n     // Default to `['v1beta1']`.\n     // +optional\n     AdmissionReviewVersions []string `json:\"admissionReviewVersions,omitempty\" protobuf:\"bytes,9,rep,name=admissionReviewVersions\"`\n}\n```\n\n### Convert to webhook-requested version\n\nWebhooks currently register to intercept particular API group/version/resource combinations.\n\nSome resources can be accessed via different versions, or even different API \ngroups (for example, `apps/v1` and `extensions/v1beta1` Deployments). To \nintercept a resource effectively, all accessible groups/versions/resources\nmust be registered for and understood by the webhook.\n\nWhen upgrading to a new version of the apiserver, existing resources can be \nmade available via new versions (or even new groups). Ensuring all webhooks\n(and registered webhook configurations) have been updated to be able to \nhandle the new versions/groups in every upgrade is possible, but easy to \nforget to do, or to do incorrectly. In the case of webhooks not authored \nby the cluster-administrator, obtaining updated admission plugins that \nunderstand the new versions could require significant effort and time.\n\nSince the apiserver can convert between all of the versions by which a resource \nis made available, this situation can be improved by having the apiserver \nconvert resources to the group/versions a webhook registered for.\n\nBecause admission can be used for out-of-tree defaulting and field enforcement,\nadmission plugins may intentionally target specific versions of resources.\nA `matchPolicy` field will be added to the webhook configuration object,\nallowing a configuration to specify whether the apiserver should only route requests\nwhich exactly match the specified rules to the webhook, or whether it should route\nrequests for equivalent resources via different API groups or versions as well.\nFor safety, this field defaults to `Exact` in `v1beta1`. In `v1`, we can default it to `Equivalent`.\n\n```golang\n// Webhook describes an admission webhook and the resources and operations it applies to.\ntype {Validating,Mutating}Webhook struct {\n     ...\n     // matchPolicy defines how the \"rules\" field is applied when a request is made \n     // to a different API group or version of a resource listed in \"rules\".\n     // Allowed values are \"Exact\" or \"Equivalent\".\n     // - Exact: match requests only if they exactly match a given rule. For example, if an object can be modified\n     // via API versions v1 and v2, and \"rules\" only includes \"v1\", do not send a request to \"v2\" to the webhook.\n     // - Equivalent: match requests if they modify a resource listed in rules via another API group or version.\n     // For example, if an object can be modified via API versions v1 and v2, and \"rules\" only includes \"v1\",\n     // a request to \"v2\" should be converted to \"v1\" and sent to the webhook.\n     // Defaults to \"Exact\"\n     // +optional\n     MatchPolicy *MatchPolicyType `json:\"matchPolicy,omitempty\"`\n```\n\nThe apiserver will do the following:\n\n1. For each resource, compute the set of other resources that access or affect the same data, and the kind of the expected object. For example:\n  * `apps,v1,deployments` (`apiVersion: apps/v1, kind: Deployment`) is also available via:\n    * `apps,v1beta2,deployments` (`apiVersion: apps/v1beta2, kind: Deployment`)\n    * `apps,v1beta1,deployments` (`apiVersion: apps/v1beta1, kind: Deployment`)\n    * `extensions,v1beta1,deployments` (`apiVersion: extensions/v1beta1, kind: Deployment`)\n  * `apps,v1,deployments/scale` (`apiVersion: autoscaling/v1, kind: Scale`) is also available via:\n    * `apps,v1beta2,deployments/scale` (`apiVersion: apps/v1beta2, kind: Scale`)\n    * `apps,v1beta1,deployments/scale` (`apiVersion: apps/v1beta1, kind: Scale`)\n    * `extensions,v1beta1,deployments/scale` (`apiVersion: extensions/v1beta1, kind: Scale`)\n2. When evaluating whether to dispatch an incoming request to a webhook with \n`matchPolicy: Equivalent`, check the request's resource *and* all equivalent \nresources against the ones the webhook had registered for. If needed, convert \nthe incoming object to one the webhook indicated it understood.\n\nThe `AdmissionRequest` sent to a webhook includes the fully-qualified\nkind (group/version/kind) and resource (group/version/resource):\n\n```golang\ntype AdmissionRequest struct {\n     ...\n     // Kind is the type of object being manipulated.  For example: Pod\n     Kind metav1.GroupVersionKind `json:\"kind\" protobuf:\"bytes,2,opt,name=kind\"`\n     // Resource is the name of the resource being requested.  This is not the kind.  For example: pods\n     Resource metav1.GroupVersionResource `json:\"resource\" protobuf:\"bytes,3,opt,name=resource\"`\n     // SubResource is the name of the subresource being requested.  This is a different resource, scoped to the parent\n     // resource, but it may have a different kind. For instance, /pods has the resource \"pods\" and the kind \"Pod\", while\n     // /pods/foo/status has the resource \"pods\", the sub resource \"status\", and the kind \"Pod\" (because status operates on\n     // pods). The binding resource for a pod though may be /pods/foo/binding, which has resource \"pods\", subresource\n     // \"binding\", and kind \"Binding\".\n     // +optional\n     SubResource string `json:\"subResource,omitempty\" protobuf:\"bytes,4,opt,name=subResource\"`\n```\n\nPrior to this conversion feature, the resource and kind of the request made to the \nAPI server, and the resource and kind sent in the AdmissionRequest were identical.\n\nWhen a conversion occurs and the object we send to the webhook is a different kind\nthan was sent to the API server, or the resource the webhook registered for is different\nthan the request made to the API server, we have three options for communicating that to the webhook:\n1. Do not expose that fact to the webhook:\n  * Set AdmissionRequest `kind` to the converted kind\n  * Set AdmissionRequest `resource` to the registered-for resource\n2. Expose that fact to the webhook using the existing fields:\n  * Set AdmissionRequest `kind` to the API request's kind (not matching the object in the AdmissionRequest)\n  * Set AdmissionRequest `resource` to the API request's resource (not matching the registered-for resource)\n3. Expose that fact to the webhook using new AdmissionRequest fields:\n  * Set AdmissionRequest `kind` to the converted kind\n  * Set AdmissionRequest `requestKind` to the API request's kind\n  * Set AdmissionRequest `resource` to the registered-for resource\n  * Set AdmissionRequest `requestResource` to the API request's resource\n\nOption 1 loses information the webhook could use (for example, to enforce different validation or defaulting rules for newer APIs).\n\nOption 2 risks breaking webhook logic by sending it resources it did not register for, and kinds it did not expect.\n\nOption 3 is preferred, and is the safest option that preserves information for use by the webhook.\n\nTo support this, three fields will be added to AdmissionRequest, and populated with the original request's kind, resource, and subResource:\n\n```golang\ntype AdmissionRequest struct {\n     ...\n     // RequestKind is the type of object being manipulated by the the original API request.  For example: Pod\n     // If this differs from the value in \"kind\", an equivalent match and conversion was performed.\n     // See documentation for the \"matchPolicy\" field in the webhook configuration type.\n     // +optional\n     RequestKind *metav1.GroupVersionKind `json:\"requestKind,omitempty\"`\n     // RequestResource is the name of the resource being requested by the the original API request.  This is not the kind.  For example: \"\"/v1/pods\n     // If this differs from the value in \"resource\", an equivalent match and conversion was performed.\n     // See documentation for the \"matchPolicy\" field in the webhook configuration type.\n     // +optional\n     RequestResource *metav1.GroupVersionResource `json:\"requestResource,omitempty\"`\n     // RequestSubResource is the name of the subresource being requested by the the original API request.  This is a different resource, scoped to the parent\n     // resource, but it may have a different kind. For instance, /pods has the resource \"pods\" and the kind \"Pod\", while\n     // /pods/foo/status has the resource \"pods\", the sub resource \"status\", and the kind \"Pod\" (because status operates on\n     // pods). The binding resource for a pod though may be /pods/foo/binding, which has resource \"pods\", subresource\n     // \"binding\", and kind \"Binding\".\n     // If this differs from the value in \"subResource\", an equivalent match and conversion was performed.\n     // See documentation for the \"matchPolicy\" field in the webhook configuration type.\n     // +optional\n     RequestSubResource string `json:\"requestSubResource,omitempty\"`\n```\n\n## V1 API\n\nThe currently planned `v1` API is described in this section.\nPlease note that as long as there are open questions in the [Graduation Criteria](#graduation-criteria) section, this is not final.\n\n```golang\npackage v1\n\ntype ScopeType string\n\nconst (\n     // ClusterScope means that scope is limited to cluster-scoped objects.\n     // Namespace API objects are cluster-scoped.\n     ClusterScope ScopeType = \"Cluster\"\n     // NamespacedScope means that scope is limited to namespaced objects.\n     NamespacedScope ScopeType = \"Namespaced\"\n     // AllScopes means that all scopes are included.\n     AllScopes ScopeType = \"*\"\n)\n\n// Rule is a tuple of APIGroups, APIVersion, and Resources.It is recommended\n// to make sure that all the tuple expansions are valid.\ntype Rule struct {\n     // APIGroups is the API groups the resources belong to. '*' is all groups.\n     // If '*' is present, the length of the slice must be one.\n     // Required.\n     APIGroups []string `json:\"apiGroups,omitempty\" protobuf:\"bytes,1,rep,name=apiGroups\"`\n\n     // APIVersions is the API versions the resources belong to. '*' is all versions.\n     // If '*' is present, the length of the slice must be one.\n     // Required.\n     APIVersions []string `json:\"apiVersions,omitempty\" protobuf:\"bytes,2,rep,name=apiVersions\"`\n\n     // Resources is a list of resources this rule applies to.\n     //\n     // For example:\n     // 'pods' means pods.\n     // 'pods/log' means the log subresource of pods.\n     // '*' means all resources, but not subresources.\n     // 'pods/*' means all subresources of pods.\n     // '*/scale' means all scale subresources.\n     // '*/*' means all resources and their subresources.\n     //\n     // If wildcard is present, the validation rule will ensure resources do not\n     // overlap with each other.\n     //\n     // Depending on the enclosing object, subresources might not be allowed.\n     // Required.\n     Resources []string `json:\"resources,omitempty\" protobuf:\"bytes,3,rep,name=resources\"`\n\n     // Scope specifies the scope of this rule.\n     // Valid values are \"Cluster\", \"Namespaced\", and \"*\"\n     // \"Cluster\" means that only cluster-scoped resources will match this rule.\n     // Namespace API objects are cluster-scoped.\n     // \"Namespaced\" means that only namespaced resources will match this rule.\n     // \"*\" means that there are no scope restrictions.\n     // Default is \"*\".\n     //\n     // +optional\n     Scope ScopeType `json:\"scope,omitempty\" protobuf:\"bytes,3,opt,name=scope\"`\n}\n\ntype ConversionPolicyType string\n\nconst (\n     // ConversionIgnore means that requests that do not match a webhook's rules but could be \n     // converted to a resource the webhook registered for, should be ignored.\n     ConversionIgnore ConversionPolicyType = \"Ignore\"\n     // ConversionConvert means that requests that do not match a webhook's rules but could be \n     // converted to a resource the webhook registered for, should be converted and sent to the webhook.\n     ConversionConvert ConversionPolicyType = \"Convert\"\n)\n\ntype FailurePolicyType string\n\nconst (\n     // Ignore means that an error calling the webhook is ignored.\n     Ignore FailurePolicyType = \"Ignore\"\n     // Fail means that an error calling the webhook causes the admission to fail.\n     Fail FailurePolicyType = \"Fail\"\n)\n\ntype SideEffectClass string\n\nconst (\n     // SideEffectClassUnknown means that no information is known about the side effects of calling the webhook.\n     // If a request with the dry-run attribute would trigger a call to this webhook, the request will instead fail.\n     SideEffectClassUnknown SideEffectClass = \"Unknown\"\n     // SideEffectClassNone means that calling the webhook will have no side effects.\n     SideEffectClassNone SideEffectClass = \"None\"\n     // SideEffectClassSome means that calling the webhook will possibly have side effects.\n     // If a request with the dry-run attribute would trigger a call to this webhook, the request will instead fail.\n     SideEffectClassSome SideEffectClass = \"Some\"\n     // SideEffectClassNoneOnDryRun means that calling the webhook will possibly have side effects, but if the\n     // request being reviewed has the dry-run attribute, the side effects will be suppressed.\n     SideEffectClassNoneOnDryRun SideEffectClass = \"NoneOnDryRun\"\n)\n\n// +genclient\n// +genclient:nonNamespaced\n// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object\n\n// ValidatingWebhookConfiguration describes the configuration of and admission webhook that accept or reject and object without changing it.\ntype ValidatingWebhookConfiguration struct {\n     metav1.TypeMeta `json:\",inline\"`\n     // Standard object metadata; More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata.\n     // +optional\n     metav1.ObjectMeta `json:\"metadata,omitempty\" protobuf:\"bytes,1,opt,name=metadata\"`\n     // Webhooks is a list of webhooks and the affected resources and operations.\n     // +optional\n     // +patchMergeKey=name\n     // +patchStrategy=merge\n     Webhooks []ValidatingWebhook `json:\"webhooks,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"name\" protobuf:\"bytes,2,rep,name=Webhooks\"`\n}\n\n// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object\n\n// ValidatingWebhookConfigurationList is a list of ValidatingWebhookConfiguration.\ntype ValidatingWebhookConfigurationList struct {\n     metav1.TypeMeta `json:\",inline\"`\n     // Standard list metadata.\n     // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\n     // +optional\n     metav1.ListMeta `json:\"metadata,omitempty\" protobuf:\"bytes,1,opt,name=metadata\"`\n     // List of ValidatingWebhookConfiguration.\n     Items []ValidatingWebhookConfiguration `json:\"items\" protobuf:\"bytes,2,rep,name=items\"`\n}\n\n// ValidatingWebhook describes an admission ValidatingWebhook and the resources and operations it applies to.\ntype ValidatingWebhook struct {\n     // The name of the admission webhook.\n     // Name should be fully qualified, e.g., imagepolicy.kubernetes.io, where\n     // \"imagepolicy\" is the name of the webhook, and kubernetes.io is the name\n     // of the organization.\n     // Required.\n     Name string `json:\"name\" protobuf:\"bytes,1,opt,name=name\"`\n\n     // ClientConfig defines how to communicate with the hook.\n     // Required\n     ClientConfig WebhookClientConfig `json:\"clientConfig\" protobuf:\"bytes,2,opt,name=clientConfig\"`\n\n     // Rules describes what operations on what resources/subresources the webhook cares about.\n     // The webhook cares about an operation if it matches _any_ Rule.\n     // However, in order to prevent ValidatingAdmissionWebhooks and MutatingAdmissionWebhooks\n     // from putting the cluster in a state which cannot be recovered from without completely\n     // disabling the plugin, ValidatingAdmissionWebhooks and MutatingAdmissionWebhooks are never called\n     // on admission requests for ValidatingWebhookConfiguration and MutatingWebhookConfiguration objects.\n     Rules []RuleWithOperations `json:\"rules,omitempty\" protobuf:\"bytes,3,rep,name=rules\"`\n\n     // matchPolicy defines how the \"rules\" field is applied when a request is made \n     // to a different API group or version of a resource listed in \"rules\".\n     // Allowed values are \"Exact\" or \"Equivalent\".\n     // - Exact: match requests only if they exactly match a given rule. For example, if an object can be modified\n     // via API version v1 and v2, and \"rules\" only includes \"v1\", do not send a request to \"v2\" to the webhook.\n     // - Equivalent: match requests if they modify a resource listed in rules via another API group or version.\n     // For example, if an object can be modified via API version v1 and v2, and \"rules\" only includes \"v1\",\n     // a request to \"v2\" should be converted to \"v1\" and sent to the webhook.\n     // Defaults to \"Equivalent\"\n     // +optional\n     MatchPolicy *MatchPolicyType `json:\"matchPolicy,omitempty\"`\n\n     // FailurePolicy defines how unrecognized errors from the admission endpoint are handled -\n     // allowed values are Ignore or Fail. Defaults to Ignore.\n     // +optional\n     FailurePolicy *FailurePolicyType `json:\"failurePolicy,omitempty\" protobuf:\"bytes,4,opt,name=failurePolicy,casttype=FailurePolicyType\"`\n    \n     // NamespaceSelector decides whether to run the webhook on an object based\n     // on whether the namespace for that object matches the selector. If the\n     // object itself is a namespace, the matching is performed on\n     // object.metadata.labels. If the object is another cluster scoped resource,\n     // it never skips the webhook.\n     //\n     // For example, to run the webhook on any objects whose namespace is not\n     // associated with \"runlevel\" of \"0\" or \"1\";  you will set the selector as\n     // follows:\n     // \"namespaceSelector\": {\n     //   \"matchExpressions\": [\n     //     {\n     //       \"key\": \"runlevel\",\n     //       \"operator\": \"NotIn\",\n     //       \"values\": [\n     //      \"0\",\n     //      \"1\"\n     //       ]\n     //     }\n     //   ]\n     // }\n     //\n     // If instead you want to only run the webhook on any objects whose\n     // namespace is associated with the \"environment\" of \"prod\" or \"staging\";\n     // you will set the selector as follows:\n     // \"namespaceSelector\": {\n     //   \"matchExpressions\": [\n     //     {\n     //       \"key\": \"environment\",\n     //       \"operator\": \"In\",\n     //       \"values\": [\n     //      \"prod\",\n     //      \"staging\"\n     //       ]\n     //     }\n     //   ]\n     // }\n     //\n     // See\n     // https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/\n     // for more examples of label selectors.\n     //\n     // Default to the empty LabelSelector, which matches everything.\n     // +optional\n     NamespaceSelector *metav1.LabelSelector `json:\"namespaceSelector,omitempty\" protobuf:\"bytes,5,opt,name=namespaceSelector\"`\n\n     // SideEffects states whether this webhookk has side effects.\n     // Acceptable values are: Unknown, None, Some, NoneOnDryRun\n     // Webhooks with side effects MUST implement a reconciliation system, since a request may be\n     // rejected by a future step in the admission change and the side effects therefore need to be undone.\n     // Requests with the dryRun attribute will be auto-rejected if they match a webhook with\n     // sideEffects == Unknown or Some.\n     SideEffects SideEffectClass `json:\"sideEffects\" protobuf:\"bytes,6,opt,name=sideEffects,casttype=SideEffectClass\"`\n\n     // ObjectSelector decides whether to run the webhook based on if the\n     // object has matching labels. objectSelector is evaluated against both\n     // the oldObject and newObject that would be sent to the webhook, and\n     // is considered to match if either object matches the selector. A null\n     // object (oldObject in the case of create, or newObject in the case of\n     // delete) or an object that cannot have labels (like a\n     // DeploymentRollback or a PodProxyOptions object) is not considered to\n     // match.\n     // Use the object selector only if the webhook is opt-in, because end\n     // users may skip the admission webhook by setting the labels.\n     // Default to the empty LabelSelector, which matches everything.\n     // +optional\n     ObjectSelector *metav1.LabelSelector `json:\"objectSelector,omitempty\"`\n\n     // TimeoutSeconds specifies the timeout for this webhook. After the timeout passes,\n     // the webhook call will be ignored or the API call will fail based on the\n     // failure policy.\n     // The timeout value must be between 1 and 30 seconds.\n     // Default to 10 seconds.\n     // +optional\n     TimeoutSeconds *int32 `json:\"timeout,omitempty\" protobuf:\"varint,8,opt,name=timeout\"`\n\n     // AdmissionReviewVersions is an ordered list of preferred `AdmissionReview`\n     // versions the Webhook expects. API server will try to use first version in\n     // the list which it supports. If none of the versions specified in this list\n     // supported by API server, validation will fail for this object.\n     // If the webhook configuration has already been persisted, calls to the \n     // webhook will fail and be subject to the failure policy.\n     // This field is required and cannot be empty.\n     AdmissionReviewVersions []string `json:\"admissionReviewVersions\" protobuf:\"bytes,9,rep,name=admissionReviewVersions\"`\n}\n\n// +genclient\n// +genclient:nonNamespaced\n// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object\n\n// MutatingWebhookConfiguration describes the configuration of and admission webhook that accept or reject and may change the object.\ntype MutatingWebhookConfiguration struct {\n     metav1.TypeMeta `json:\",inline\"`\n     // Standard object metadata; More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata.\n     // +optional\n     metav1.ObjectMeta `json:\"metadata,omitempty\" protobuf:\"bytes,1,opt,name=metadata\"`\n     // Webhooks is a list of webhooks and the affected resources and operations.\n     // +optional\n     // +patchMergeKey=name\n     // +patchStrategy=merge\n     Webhooks []MutatingWebhook `json:\"webhooks,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"name\" protobuf:\"bytes,2,rep,name=Webhooks\"`\n}\n\n// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object\n\n// MutatingWebhookConfigurationList is a list of MutatingWebhookConfiguration.\ntype MutatingWebhookConfigurationList struct {\n     metav1.TypeMeta `json:\",inline\"`\n     // Standard list metadata.\n     // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\n     // +optional\n     metav1.ListMeta `json:\"metadata,omitempty\" protobuf:\"bytes,1,opt,name=metadata\"`\n     // List of MutatingWebhookConfiguration.\n     Items []MutatingWebhookConfiguration `json:\"items\" protobuf:\"bytes,2,rep,name=items\"`\n}\n\n// MutatingWebhook describes an admission MutatingWebhook and the resources and operations it applies to.\ntype MutatingWebhook struct {\n     // The name of the admission webhook.\n     // Name should be fully qualified, e.g., imagepolicy.kubernetes.io, where\n     // \"imagepolicy\" is the name of the webhook, and kubernetes.io is the name\n     // of the organization.\n     // Required.\n     Name string `json:\"name\" protobuf:\"bytes,1,opt,name=name\"`\n\n     // ClientConfig defines how to communicate with the hook.\n     // Required\n     ClientConfig WebhookClientConfig `json:\"clientConfig\" protobuf:\"bytes,2,opt,name=clientConfig\"`\n\n     // Rules describes what operations on what resources/subresources the webhook cares about.\n     // The webhook cares about an operation if it matches _any_ Rule.\n     // However, in order to prevent ValidatingAdmissionWebhooks and MutatingAdmissionWebhooks\n     // from putting the cluster in a state which cannot be recovered from without completely\n     // disabling the plugin, ValidatingAdmissionWebhooks and MutatingAdmissionWebhooks are never called\n     // on admission requests for ValidatingWebhookConfiguration and MutatingWebhookConfiguration objects.\n     Rules []RuleWithOperations `json:\"rules,omitempty\" protobuf:\"bytes,3,rep,name=rules\"`\n\n     // matchPolicy defines how the \"rules\" field is applied when a request is made \n     // to a different API group or version of a resource listed in \"rules\".\n     // Allowed values are \"Exact\" or \"Equivalent\".\n     // - Exact: match requests only if they exactly match a given rule. For example, if an object can be modified\n     // via API version v1 and v2, and \"rules\" only includes \"v1\", do not send a request to \"v2\" to the webhook.\n     // - Equivalent: match requests if they modify a resource listed in rules via another API group or version.\n     // For example, if an object can be modified via API version v1 and v2, and \"rules\" only includes \"v1\",\n     // a request to \"v2\" should be converted to \"v1\" and sent to the webhook.\n     // Defaults to \"Equivalent\"\n     // +optional\n     MatchPolicy *MatchPolicyType `json:\"matchPolicy,omitempty\"`\n\n     // FailurePolicy defines how unrecognized errors from the admission endpoint are handled -\n     // allowed values are Ignore or Fail. Defaults to Ignore.\n     // +optional\n     FailurePolicy *FailurePolicyType `json:\"failurePolicy,omitempty\" protobuf:\"bytes,4,opt,name=failurePolicy,casttype=FailurePolicyType\"`\n\n     // reinvocationPolicy indicates whether this webhook should be called multiple times as part of a single admission evaluation.\n     // Allowed values are \"Never\" and \"IfNeeded\".\n     //\n     // Never: the webhook will not be called more than once in a single admission evaluation.\n     //\n     // IfNeeded: the webhook will be called at least one additional time as part of the admission evaluation\n     // if the object being admitted is modified by other admission plugins after the initial webhook call.\n     // Webhooks that specify this option *must* be idempotent, able to process objects they previously admitted.\n     // Note:\n     // * the number of additional invocations is not guaranteed to be exactly one.\n     // * if additional invocations result in further modifications to the object, webhooks are not guaranteed to be invoked again.\n     // * webhooks that use this option may be reordered to minimize the number of additional invocations.\n     // * to validate an object after all mutations are guaranteed complete, use a validating admission webhook instead.\n     //\n     // Defaults to \"Never\".\n     // +optional\n     ReinvocationPolicy *ReinvocationPolicyType `json:\"reinvocationPolicy,omitempty\"`\n    \n     // NamespaceSelector decides whether to run the webhook on an object based\n     // on whether the namespace for that object matches the selector. If the\n     // object itself is a namespace, the matching is performed on\n     // object.metadata.labels. If the object is another cluster scoped resource,\n     // it never skips the webhook.\n     //\n     // For example, to run the webhook on any objects whose namespace is not\n     // associated with \"runlevel\" of \"0\" or \"1\";  you will set the selector as\n     // follows:\n     // \"namespaceSelector\": {\n     //   \"matchExpressions\": [\n     //     {\n     //       \"key\": \"runlevel\",\n     //       \"operator\": \"NotIn\",\n     //       \"values\": [\n     //      \"0\",\n     //      \"1\"\n     //       ]\n     //     }\n     //   ]\n     // }\n     //\n     // If instead you want to only run the webhook on any objects whose\n     // namespace is associated with the \"environment\" of \"prod\" or \"staging\";\n     // you will set the selector as follows:\n     // \"namespaceSelector\": {\n     //   \"matchExpressions\": [\n     //     {\n     //       \"key\": \"environment\",\n     //       \"operator\": \"In\",\n     //       \"values\": [\n     //      \"prod\",\n     //      \"staging\"\n     //       ]\n     //     }\n     //   ]\n     // }\n     //\n     // See\n     // https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/\n     // for more examples of label selectors.\n     //\n     // Default to the empty LabelSelector, which matches everything.\n     // +optional\n     NamespaceSelector *metav1.LabelSelector `json:\"namespaceSelector,omitempty\" protobuf:\"bytes,5,opt,name=namespaceSelector\"`\n\n     // SideEffects states whether this webhookk has side effects.\n     // Acceptable values are: Unknown, None, Some, NoneOnDryRun\n     // Webhooks with side effects MUST implement a reconciliation system, since a request may be\n     // rejected by a future step in the admission change and the side effects therefore need to be undone.\n     // Requests with the dryRun attribute will be auto-rejected if they match a webhook with\n     // sideEffects == Unknown or Some.\n     SideEffects SideEffectClass `json:\"sideEffects\" protobuf:\"bytes,6,opt,name=sideEffects,casttype=SideEffectClass\"`\n\n     // ObjectSelector decides whether to run the webhook based on if the\n     // object has matching labels. objectSelector is evaluated against both\n     // the oldObject and newObject that would be sent to the webhook, and\n     // is considered to match if either object matches the selector. A null\n     // object (oldObject in the case of create, or newObject in the case of\n     // delete) or an object that cannot have labels (like a\n     // DeploymentRollback or a PodProxyOptions object) is not considered to\n     // match.\n     // Use the object selector only if the webhook is opt-in, because end\n     // users may skip the admission webhook by setting the labels.\n     // Default to the empty LabelSelector, which matches everything.\n     // +optional\n     ObjectSelector *metav1.LabelSelector `json:\"objectSelector,omitempty\"`\n\n     // TimeoutSeconds specifies the timeout for this webhook. After the timeout passes,\n     // the webhook call will be ignored or the API call will fail based on the\n     // failure policy.\n     // The timeout value must be between 1 and 30 seconds.\n     // Default to 10 seconds.\n     // +optional\n     TimeoutSeconds *int32 `json:\"timeout,omitempty\" protobuf:\"varint,8,opt,name=timeout\"`\n\n     // AdmissionReviewVersions is an ordered list of preferred `AdmissionReview`\n     // versions the Webhook expects. API server will try to use first version in\n     // the list which it supports. If none of the versions specified in this list\n     // supported by API server, validation will fail for this object.\n     // If the webhook configuration has already been persisted, calls to the \n     // webhook will fail and be subject to the failure policy.\n     // This field is required and cannot be empty.\n     AdmissionReviewVersions []string `json:\"admissionReviewVersions\" protobuf:\"bytes,9,rep,name=admissionReviewVersions\"`\n}\n\n// RuleWithOperations is a tuple of Operations and Resources. It is recommended to make\n// sure that all the tuple expansions are valid.\ntype RuleWithOperations struct {\n     // Operations is the operations the admission hook cares about - CREATE, UPDATE, or *\n     // for all operations.\n     // If '*' is present, the length of the slice must be one.\n     // Required.\n     Operations []OperationType `json:\"operations,omitempty\" protobuf:\"bytes,1,rep,name=operations,casttype=OperationType\"`\n     // Rule is embedded, it describes other criteria of the rule, like\n     // APIGroups, APIVersions, Resources, etc.\n     Rule `json:\",inline\" protobuf:\"bytes,2,opt,name=rule\"`\n}\n\ntype OperationType string\n\n// The constants should be kept in sync with those defined in k8s.io/kubernetes/pkg/admission/interface.go.\nconst (\n    OperationAll OperationType = \"*\"\n     Create       OperationType = \"CREATE\"\n     Update       OperationType = \"UPDATE\"\n     Delete       OperationType = \"DELETE\"\n     Connect      OperationType = \"CONNECT\"\n)\n\n// WebhookClientConfig contains the information to make a TLS\n// connection with the webhook\ntype WebhookClientConfig struct {\n     // `url` gives the location of the webhook, in standard URL form\n     // (`scheme://host:port/path`). Exactly one of `url` or `service`\n     // must be specified.\n     //\n     // The `host` should not refer to a service running in the cluster; use\n     // the `service` field instead. The host might be resolved via external\n     // DNS in some apiservers (e.g., `kube-apiserver` cannot resolve\n     // in-cluster DNS as that would be a layering violation). `host` may\n     // also be an IP address.\n     //\n     // Please note that using `localhost` or `127.0.0.1` as a `host` is\n     // risky unless you take great care to run this webhook on all hosts\n     // which run an apiserver which might need to make calls to this\n     // webhook. Such installs are likely to be non-portable, i.e., not easy\n     // to turn up in a new cluster.\n     //\n     // A path is optional, and if present may be any string permissible in\n     // a URL. You may use the path to pass an arbitrary string to the\n     // webhook, for example, a cluster identifier.\n     //\n     // Attempting to use a user or basic auth e.g. \"user:password@\" is not\n     // allowed. Fragments (\"#...\") and query parameters (\"?...\") are not\n     // allowed, either.\n     //\n     // +optional\n     URL *string `json:\"url,omitempty\" protobuf:\"bytes,3,opt,name=url\"`\n\n     // `service` is a reference to the service for this webhook. Either\n     // `service` or `url` must be specified.\n     //\n     // If the webhook is running within the cluster, then you should use `service`.\n     //\n     // +optional\n     Service *ServiceReference `json:\"service,omitempty\" protobuf:\"bytes,1,opt,name=service\"`\n\n     // `caBundle` is a PEM encoded CA bundle which will be used to validate the webhook's server certificate.\n     // If unspecified, system trust roots on the apiserver are used.\n     // +optional\n     CABundle []byte `json:\"caBundle,omitempty\" protobuf:\"bytes,2,opt,name=caBundle\"`\n}\n\n// ServiceReference holds a reference to Service.legacy.k8s.io\ntype ServiceReference struct {\n     // `namespace` is the namespace of the service.\n     // Required\n     Namespace string `json:\"namespace\" protobuf:\"bytes,1,opt,name=namespace\"`\n     // `name` is the name of the service.\n     // Required\n     Name string `json:\"name\" protobuf:\"bytes,2,opt,name=name\"`\n\n     // `path` is an optional URL path which will be sent in any request to\n     // this service.\n     // +optional\n     Path *string `json:\"path,omitempty\" protobuf:\"bytes,3,opt,name=path\"`\n\n     // If specified, the port on the service that hosting webhook.\n     // Default to 443 for backward compatibility.\n     // `Port` should be a valid port number (1-65535, inclusive).\n     // +optional\n     Port *int32 `json:\"port,omitempty\" protobuf:\"varint,4,opt,name=port\"`\n}\n```\n\n## V1beta1 changes\n\nAll of the proposed changes will be added to `v1beta1` for backward compatibility \nand also to keep roundtrip-ability between `v1` and `v1beta1`. The only difference are:\n\n* Default Value for `timeoutSeconds` field will be 30 seconds for `v1beta1`.\n* `AdmissionReviewVersions` list is optional in v1beta1 and defaulted to `['v1beta1']` while required in `v1`.\n\n## Validations\n\nThese set of new validation will be applied to both v1 and v1beta1:\n\n* `Scope` field can only have `Cluster`, `Namespaced`, or `*` values (if empty, the field defaults to `*`).\n* `Timeout` field must be between 1 and 30 seconds.\n* `AdmissionReviewVersions` list must have at least one version supported by the API Server serving it. Note that for downgrade compatibility, Webhook authors should always support as many `AdmissionReview` versions as possible.\n\n## Risks and Mitigations\n\nThe features proposed in this KEP are low risk and mostly bug fixes or new features that should have little to no risk on existing features.\n\n## Graduation Criteria\n\nTo mark these as complete, all of the above features need to be implemented. \nAn [umbrella issue](https://github.com/kubernetes/kubernetes/issues/73185) is tracking all of these changes.\nAlso there need to be sufficient tests for any of these new features and all existing features and documentation should be completed for all features.\n\nThere are still open questions that need to be addressed and updated in this KEP before graduation:\n\n## Post-GA tasks\n\nThese are related Post-GA tasks:\n\n* Allow mutating of existing objects in `DELETE` events\n  * Should adding/removing existing finalizers in delete admission be allowed?\n  * Should GC finalizers based on `DeleteOptions` happen before or after mutating admission?\n* If we have enough evidence, consider a better solution for plugin ordering like letting a webhook indicate how many times it should be rerun.\n\n## Implementation History\n\n* First version of the KEP being merged: Jan 29th, 2019\n* The set of features for GA approved, initial set of features marked implementable: Feb 4th, 2019\n* Implementation start for all approved changes: Feb 4th, 2019\n* Added details for auto-conversion: Apr 25th, 2019\n* Added details for mutating admission re-running: May 6th, 2019\n"
+  },
+  {
+    "id": "491d189fa6c6e2f67adab8f11b306c5b",
+    "title": "Publish CRD OpenAPI",
+    "authors": ["@roycaihw"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": null,
+    "reviewers": ["@apelisse", "@liggitt", "@mbohlool", "@sttts"],
+    "approvers": ["@liggitt", "@sttts"],
+    "editor": "TBD",
+    "creationDate": "2019-02-07",
+    "lastUpdated": "2019-02-13",
+    "status": "implementable",
+    "seeAlso": [
+      "[Validation for CustomResources design doc](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/customresources-validation.md)"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Publish CRD OpenAPI\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Build Schema/Definition](#build-schemadefinition)\n  - [Build Spec](#build-spec)\n  - [Aggregate and Publish Spec from apiextensions-apiserver](#aggregate-and-publish-spec-from-apiextensions-apiserver)\n  - [Aggregate and Publish Spec from kube-aggregator](#aggregate-and-publish-spec-from-kube-aggregator)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nIn CustomResourceDefinition (CRD) we allow CRD author to define OpenAPI v3 schema, to\nenable server-side [validation for CustomResources (CR)](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/customresources-validation.md).\nThe validation schema format is compatible for creating OpenAPI documentation for CRs,\nwhich can be used by clients like kubectl to perform client-side validation\n(e.g. `kubectl create` and `kubectl apply`),\nschema explanation (`kubectl explain`), and client generation.\nThis KEP proposes using the OpenAPI v3 schema to create and publish OpenAPI\ndocumentation for CRs.\n\n## Motivation\n\nPublishing CRD OpenAPI enables client-side validation, schema explanation and\nclient generation for CRs. It covers the gap between CR and native Kubernetes\nAPIs, which already support OpenAPI documentation.\n\nPublishing CRD OpenAPI is also noted as potential followup in [Validation for CustomResources Implementation Plan](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/customresources-validation.md#implementation-plan).\n\n### Goals\n\n* For every CRD that we serve, publish Paths (operations that we support on\n  resource and subresources) and Definitions (for both CR object and CR list\n  object) in OpenAPI documentation to fully demonstrate the existence of the\n  API.\n* For CRDs with schema defined, the CR object Definition should\n  include both CRD schema and native Kubernetes ObjectMeta and\n  TypeMeta properties.\n* For CRDs without schema, the CR object definition will be as\n  complete as possible while still maintaining compatibility with the openapi\n  spec and with supported kubernetes components\n\n### Non-Goals\n\n* Finalize design on how to aggregate and publish CRD OpenAPI from kube-aggregator\n\n## Proposal\n\nRef: https://github.com/kubernetes/kubernetes/pull/71192\n\n### Build Schema/Definition\n\nWe drop the features that OpenAPI v2 don't support from CRD OpenAPI v3 schema\n(listed in this [spread-sheet](https://docs.google.com/spreadsheets/d/1Mkm9L7CXGvRorV0Cr4Vwfu0DH7XRi24YHPiDK1NZWo4/edit?usp=sharing)\nby @nikhita and @sttts: oneOf, anyOf and not) (proposed [here](https://github.com/kubernetes/kubernetes/issues/49879#issuecomment-320031200)).\n\nWe add documentation in CRD API pointing it out to CRD authors that if they want\na complete and valid OpenAPI spec being generated for their Resource, they should\nnot use v3 features like `oneOf` (proposed [here](https://github.com/kubernetes/kubernetes/issues/49879#issuecomment-321774254)).\n\nFor CRD with schema, we append Kubernetes TypeMeta and ObjectMeta to the schema.\n\n### Build Spec\n\nWe template a WebService object for each CRD like what we do for [built-in APIs](https://github.com/kubernetes/kubernetes/blob/8b98e802eddb9f478ff7d991a2f72f60c165388a/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go#L565-L845),\nfeed the WebService to kube-openapi builder to build OpenAPI spec.\n\n### Aggregate and Publish Spec from apiextensions-apiserver\n\nWe add a spec aggregator to apiextensions-apiserver like [kube-aggregator\ndoes](https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/aggregator.go).\nThe aggregator keeps a {CR GroupVersionKind (GVK) -\u003e OpenAPI spec} mapping,\naggregates specs, and serves the spec through \"/openapi/v2\" handler from\napiextensions-apiserver.\n\nAll the code mentioned above (CRD OpenAPI builder, aggregator and controller)\nlive within apiextensions-apiserver staging repo and get called within the\nsame repo.\n\n### Aggregate and Publish Spec from kube-aggregator\n\nWe add apiextensions-apiserver into the resync queue in kube-aggregator openapi\ncontroller with a resync period of 1 second. Previously apiextensions-apiserver\n(delegation target) was treated as static and never get resynced.\n\nDownloading spec and etag from apiextensions-apiserver is in-process function\ncall. We skip aggregation when etag doesn't change. So we are not adding much\nload.\n\nAlternative solutions discussed in [this doc](https://docs.google.com/document/d/13lBj8Stdwku8BgL0fbT__4Iw97NRh77loJ_MoZuCGwQ/edit#). Finalizing the design is a non-goal for 1.14.\n\n### Risks and Mitigations\n\nPerformance: OpenAPI aggregation has performed poorly in the past. Introducing\naggregation that occurs on a higher frequency could cause performance issues in\nthe apiserver. To mitigate this, improvements in OpenAPI merging are being made\n(both in terms of memory and CPU cost), and benchmarking will be done prior to\nmerge to ensure we do not regress API server performance.\n\nCompatibility: some Kubernetes components (notably kubectl) perform client-side\nvalidation based on published openapi schemas. To ensure that publishing schemas\nfor additional custom resources does not regress existing function, we will test\ncreation of custom resources with and without validation with current and prior versions of kubectl.\n\nWe will add a feature gate CustomResourcePublishOpenAPI, which can be turned on to start CRD\nopenapi creation \u0026 aggregation \u0026 publishing. The feature gate will be Alpha (defaulted to\nFalse) in 1.14, Beta (defaulted to True) in 1.15 and Stable in 1.16.\n\n## Graduation Criteria\n\nIn 1.14, publishing CRD OpenAPI should be e2e-tested to ensure:\n\n1. for CRD with validation schema\n\n* client-side validation (`kubectl create` and `kubectl apply`) works to:\n  * allow requests with known and required properties\n  * reject requests with unknown properties when disallowed by the schema\n  * reject requests without required properties\n\n* `kubectl explain` works to:\n  * explain CR properties\n  * explain CR properties recursively\n  * return error when explain is called on property that doesn't exist\n\n2. for CRD without validation schema\n\n* client-side validation (`kubectl create` and `kubectl apply`) works to:\n  * allow requests with any unknown properties\n\n* `kubectl explain` works to:\n  * returns an understandable error or message when asked to explain a CR with no available schema\n\n3. for any CRD with/without validation schema, we do not regress existing\n  behavior for prior version of kubectl:\n\n* `kubectl create` client-side validation works to:\n  * allow requests with any unknown properties\n\n* `kubectl explain` works to:\n  * returns an understandable error or message when asked to explain a CR with no available schema\n\n4. for multiple CRDs\n  * CRs in different groups (two CRDs) show up in OpenAPI documentation\n  * CRs in the same group but different version (one multiversion CRD or two\n    CRDs) show up in OpenAPI\n    documentation\n  * CRs in the same group version but different kind (two CRDs) show up in OpenAPI\n    documentation\n\n5. for publishing latency and cpu load\n  * does not consume significant delay to update OpenAPI Paths and Definitions\n    after CRD gets created/updated/deleted\n  * does not measurably increase server load at steady state\n\n6. for `kubectl apply`\n  * verify no regression using apply with CRs with and without schemas, on the current and previous version of kubectl\n\n## Implementation History\n\n"
+  },
+  {
+    "id": "294501fb1683b17b87fc35317eefe2e2",
+    "title": "Graduate CustomResourceDefinitions to GA",
+    "authors": ["@jpbetz", "@roycaihw", "@sttts"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["sig-api-machinery", "sig-architecture"],
+    "reviewers": ["@deads2k", "@lavalamp", "@liggitt", "@mbohlool", "@sttts"],
+    "approvers": ["@deads2k", "@lavalamp"],
+    "editor": "TBD",
+    "creationDate": "2018-04-15",
+    "lastUpdated": "2018-04-24",
+    "status": "implementable",
+    "seeAlso": [
+      "[Umbrella Issue](https://github.com/kubernetes/kubernetes/issues/58682)",
+      "[Vanilla OpenAPI Subset Design](https://docs.google.com/document/d/1pcGlbmw-2Y0JJs9hsYnSBXamgG9TfWtHY6eh80zSTd8)",
+      "[Pruning for CustomResources KEP](https://github.com/kubernetes/enhancements/pull/709)",
+      "[Defaulting for Custom Resources KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190426-crd-defaulting.md)"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Title\n\nGraduate CustomResourceDefinitions to GA\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Defaulting and pruning for custom resources is implemented](#defaulting-and-pruning-for-custom-resources-is-implemented)\n  - [CRD v1 schemas are restricted to a subset of the OpenAPI specification](#crd-v1-schemas-are-restricted-to-a-subset-of-the-openapi-specification)\n  - [Generator exists for CRD Validation Schema v3 (Kubebuilder)](#generator-exists-for-crd-validation-schema-v3-kubebuilder)\n  - [CustomResourceWebhookConversion API is GA ready](#customresourcewebhookconversion-api-is-ga-ready)\n  - [CustomResourceSubresources API is GA ready](#customresourcesubresources-api-is-ga-ready)\n  - [v1 API](#v1-api)\n- [Test Plan](#test-plan)\n  - [Integration tests for GA](#integration-tests-for-ga)\n  - [e2e tests for GA](#e2e-tests-for-ga)\n  - [Conformance tests for GA](#conformance-tests-for-ga)\n  - [Scale Targets for GA](#scale-targets-for-ga)\n- [Graduation Criteria](#graduation-criteria)\n- [Post-GA tasks](#post-ga-tasks)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nCustomResourceDefinitions (CRDs) are the way to extend the Kubernetes API to\ninclude custom resource types that behave like the native resource types. CRDs\nhave been in Beta since Kubernetes 1.7. This document outlines the required\nsteps to graduate CRDs to general availability (GA).\n\n## Motivation\n\nCRDs are in widespread use as a Kubernetes extensibility mechanism and have been\navailable in beta for the last 5 Kubernetes releases. Feature requests and bug\nreports suggest CRDs are nearing GA quality, and this KEP aims to itemize the\nremaining improvements to move this toward GA.\n\n### Goals\n\nEstablish consensus for the remaining essential improvements needed to move CRDs to GA.\n\nGuiding principles:\n* if a missing feature forces or encourages users to implement non-Kubernetes-like APIs, and therefore damages the ecosystem long term, it belongs on this list.\n* If a feature cannot be added as an after-thought of a GA API, it or some preparation belongs on this list.\n\n\n### Non-Goals\n\nFull feature parity with native kubernetes resource types is not a GA graduation goal. See above guiding principles.\n\n## Proposal\n\nObjectives to graduate CRDs to GA, each of which is described in more detail further below:\n\n* Defaulting and pruning for custom resources is implemented\n* CRD v1 schemas are restricted to a subset of the OpenAPI specification (and there is a v1beta1 compatibility plan)\n* Generator exists for CRD Validation Schema v3 (Kubebuilder)\n* CustomResourceWebhookConversion API is GA ready\n* CustomResourceSubresources API is GA ready\n\nBug fixes required to graduate CRDs to GA:\n\n* See “Required for GA” issues tracked via the [CRD Project Board](https://github.com/orgs/kubernetes/projects/28).\n\nFor additional details on already completed features, see the [CRD Project Board](https://github.com/orgs/kubernetes/projects/28).\n\nSee [Post-GA tasks](#post-ga-tasks) for decided out-of-scope features.\n\n### Defaulting and pruning for custom resources is implemented\n\nBoth defaulting and pruning and also read-only validation are blocked by the\nOpenAPI subset definition (next point). \n\nSee the [Pruning for CustomResources KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20180731-crd-pruning.md)\nand the [Defaulting for Custom Resources KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190426-crd-defaulting.md).\n\n### CRD v1 schemas are restricted to a subset of the OpenAPI specification\n\nSee [OpenAPI Subset KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190425-structural-openapi.md)\n\n### Generator exists for CRD Validation Schema v3 (Kubebuilder)\n\nSee [kubebuilder’s\ncrd-gen](https://github.com/kubernetes-sigs/controller-tools/tree/master/pkg/crd)\nand a more general\n[crd-schema-gen](https://github.com/kubernetes-sigs/controller-tools/pull/183),\nto be integrated into kubebuidler’s controller-tools.\n\n### CustomResourceWebhookConversion API is GA ready\n\nCurrently CRD webhook conversion is alpha. We plan to take this to v1beta1 according to the\n[CustomResourceDefinition Conversion Webhook's Graduation Criteria](https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190425-crd-conversion-webhook.md#graduation-criteria).\nWe plan to then graduate this to GA as part of the CRD to GA graduation.\n\n### CustomResourceSubresources API is GA ready\n\nCurrently custom resource subresources are v1beta1 and provide support for the\n/status and /scale subresources. We plan to graduate this to GA as part of the\nCRD to GA graduation.\n\n### v1 API\n\nThe CRD `v1` API will be the same as the `v1beta1` but with all changes to the API from the GA tasks:\n\n* Rename misnamed json field [JSONPath](https://github.com/kubernetes/kubernetes/blob/06bc7e3e0026ea25065f59f4bd305c0b7dbbc145/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go#L226-L227) to `jsonPath`\n* [Replace top-level fields with per version fields](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/customresource-conversion-webhook.md#top-level-fields-to-per-version-fields)\n* Restrict OpenAPI per [Vanilla OpenAPI Subset Design](https://docs.google.com/document/d/1pcGlbmw-2Y0JJs9hsYnSBXamgG9TfWtHY6eh80zSTd8)\n\n## Test Plan\n\n### Integration tests for GA\n\n* Ensure CRD themselves are v1\u003c-\u003ev1beta1 round trippable\n* CRD versioning/conversion integration test coverage:\n  * Ensure what is persisted in etcd matches the storage version\n  * Set up a CRD, persist some data, changed the version, and ensure the previously persisted data is readable\n  * Ensure discovery docs track a CRD through creation, version addition, version removal, and deletion\n  * Ensure `spec.served` is respected\n  * Ensure illegal inputs result in sensible errors, not panics, and that the system recovers appropriately\n    * CRD mutation operations: invalid schemas, misconfigured webhooks, scope changes, ...\n    * webhook conversion responses: conversion responses that do not align with request (uuid, list size and elements, ..)\n* See also [Pruning for Custom Resources KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20180731-crd-pruning.md)\n* See also [Vanilla OpenAPI Subset: structural schemas KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190425-structural-openapi.md)\n* See also [Defaulting for Custom Resources KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190426-crd-defaulting.md)\n\n### e2e tests for GA\n\n* Custom Resources should be readable and writable at all available versions (test for round trip-ability)\n* Introduce v1 API testing in addition to v1beta1 tests\n\n### Conformance tests for GA\n\n* Prepare a e2e test case candidates that we will request be promoted to Conformance (which is a process that happens after GA)\n  * test case covering the basics of the CRD v1 API\n  * test cases exemplifying CRD versioning for both `strategy: None` and `strategy: Webhook`.\n\n### Scale Targets for GA\n\nThe scale targets for GA of custom resources are defined by the same [API call latency\nSLIs/SLOs as the Kuberetes native types](https://github.com/kubernetes/community/blob/master/sig-scalability/slos/api_call_latency.md#api-call-latency-slisslos-details).\n\nThe targets are defined by the below suggested maximum limits, which are organized the same way as the [Kubernetes native type thresholds](https://github.com/kubernetes/community/blob/master/sig-scalability/configs-and-limits/thresholds.md#kubernetes-thresholds), with only one change:\n\n- Since custom resources can be arbitrarily large, we have broken down the limit by custom resource object size.\n\n**Custom Resource Definitions:**\n\n| Suggested Maximum Limit: scope=cluster |\n| 500 |\n\n_Note: The Custom Resource Definition suggested maximum limit was selected not\ndue to the above SLI/SLOs, but instead due to the latency OpenAPI publishing,\nwhich is a background process that occurs asychroniously each time a Custom\nResource Definition schema is updated. For 500 Custom Resource Definitions it takes\nslightly over 35 seconds for a definition change to be visible via the OpenAPI\nspec endpoint._\n\n**Custom Resources, Cluster Wide:**\n\nCluster wide limits for custom resources are storage bound and custom resources\nshare the storage space with all other objects. While determining the\nappropriate storage limit for a cluster is out-of-scope for this document, once\na etcd storage limit selected, suggested maximum limits for custom resources\nare:\n\n| etcd storage limit | Suggested Maximum Limit: scope=cluster |\n| 4GB | 40000 |\n| 8GB | 80000 |\n\nThese limits aim to keep custom resource storage usage to less than half of the\ntotal cluster storage capacity for custom resources of 50kb or less in size.\n\n**Custom Resources per Definition:**\n\nFor each custom resource definition, the limit on the number of custom resources\ncan be found by taking the (median) object size of the custom resource and finding\nthe the matching row in this table:\n\n| Object size | Suggested Maximum Limit: scope=namespace (5s p99 SLO) | Suggested Maximum Limit: scope=cluster (30s p99 SLO) |\n| \u003c=10kb | 1500 | 10000 |\n| (10kb - 25kb] | 600 | 4000 |\n| (25kb - 50kb] | 300 | 2000 |\n\nThe cluster scope indicates the total number of custom resources for that\ndefinition allowed in the entire cluster.\n\nThe namespace scope indicates the total number of custom resources for that\ndefinition allowed in any particular namespace. The cumulative count of the\ncustom resource across all namespaces must not exceed the cluster limit.\n\nSince, in practice, custom resources scale farther without conversion webhooks\nwithin the SLI/SLOs (roughly 2x according to our scale tests), custom resource\ndefinition authors should be careful to adhere to these limits so that in the\nfuture a webhook converter may safely be added as part of a custom resource\nversion upgrade.\n\n_Note: For custom resources of custom resource definitions using `scope: Namespaced`: the scope=namespace\nsuggested maximum limit indicates how many custom resource objects may be in each namespace,\nand the scope=cluster suggested maximum limit indicates how many custom resource objects may\nbe in the cluster total. For custom resources of custom resource definitions using `scope: Cluster`: only\nthe scope=cluster suggested maximum limit applies._\n\n**Conversion Webhooks:**\n\nConversion Webhook SLOs are defined from the perspective of the conversion\nwebhook. It does not include any api-server serialization/deserialization for\nmaking the request to the webhook, but it does include network latency.\n\nGiven that the performance and scalability of conversion webhooks are the\nresponsibility of their author, Custom resource scale targets are applied only for\nconversion webhooks that are within the following latencies for the above suggested\nmaximum limits.\n\n| scope | object count limit | Expected conversion Webhook SLO: p99 latency |\n| resource | 1 | 50ms |\n| namespace | 1500 (\u003c=10kb), 600 (10-25kb) or 300 (25-50kb) | 1 seconds |\n| cluster | 10000 (\u003c=10kb), 4000 (10-25kb) or 2000 (25-50kb) | 6 seconds |\n\nThe scope=resource's higher \"per-object\" latency (50ms vs ~1.5ms for namespace\nand cluster scope) is to accommodate for a request serving cost constant.\n\nThe above object size and suggested maximum limits in the Custom Resources per\nDefinition table applies to these conversion webhook SLOs. For example, for a\nlist request for 1500 custom resource objects that are 10k in size, the resource\nscope SLO of 1 second for the conversion webhook applies.\n\n**Scale Target Data**\n\nGA custom resource scale targets were selected based on an [analysis of our current scale limits](https://docs.google.com/document/d/1tEstPQvzGvaRnN-WwGUWx1H9xHPRCy_fFcGlgTkB3f8).\n\nWe ran a month long survey of Custom Resource Definition scale needs across Kubernetes mailing lists, slack channels and social media.\nOf the custom resource definitions surveyed, 96% are currently within these suggested maximum limits, 91% are within these limits for their anticipated future growth, and survey data provides useful guidance for our post-GA scalability work. See [survey of real-world custom resource usage](https://docs.google.com/document/d/1MTd_gDlpgBaT5sAKM4j6tQVeCFIT9J44RHzt2yWOK_g) for details.\n\nAs part of GA the suggested maximum limits and SLO documentation will be updated to make this clear and to\nencourage CRD authors to provide concrete suggested maximum limits and SLIs/SLOs for their custom\nresource kinds to their users that account for the per resource conversion cost\nof their conversion webhook and/or size of their custom resources.\n\n## Graduation Criteria\n\nTo mark these as complete, all of the above features need to be implemented. An\numbrella issue is tracking all of these changes. Also there need to be\nsufficient tests for any of these new features and all existing features and\ndocumentation should be completed for all features.\n\nSee [umbrella issue](https://github.com/kubernetes/kubernetes/issues/58682) for status.\n\n## Post-GA tasks\n\nSee the [umbrella issue](https://github.com/kubernetes/kubernetes/issues/58682) for details on Post-GA tasks. The tasks at the time this KEP was written are:\n\n* Human readable status from conditions for a CRD using additionalPrinterColumns (https://github.com/kubernetes/kubernetes/issues/67268)\n* Consider changing the schema language in CRDs (https://github.com/kubernetes/kubernetes/issues/67840)\n* Should support graceful deletion for storage (https://github.com/kubernetes/kubernetes/issues/68508)\n* Enable arbitrary CRD field selectors by supporting a whitelist of fields in CRD spec (https://github.com/kubernetes/kubernetes/issues/53459)\n* Support graceful deletion for custom resources (https://github.com/kubernetes/kubernetes/issues/56567)\n* CRD validation webhooks (https://github.com/kubernetes/community/pull/1418)\n* Allow OpenAPI references in the CRD valiation schema (https://github.com/kubernetes/kubernetes/issues/54579, https://github.com/kubernetes/kubernetes/issues/62872)\n* Generate json-schema for use in the CRDs from the go types (https://github.com/kubernetes/kubernetes/issues/59154, https://github.com/kubernetes/sample-controller/issues/2, https://github.com/kubernetes/code-generator/issues/28)\n* Support for namespace-local CRD (https://github.com/kubernetes/kubernetes/issues/65551)\n* Support proto encoding for custom resources (https://github.com/kubernetes/kubernetes/issues/63677)\n* labelSelectorPath should be allowed not be under .status (https://github.com/kubernetes/kubernetes/issues/66688)\n* Support arbitrary non-standard subresources for CRDs (https://github.com/kubernetes/kubernetes/issues/72637)\n* OpenAPI v3 is published for all resources, including custom resources\n\n## Implementation History\n"
+  },
+  {
+    "id": "4a4e46ffac8ba8d35234eb029529bcd8",
+    "title": "Pruning for Custom Resources",
+    "authors": ["@sttts"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["sig-api-machinery"],
+    "reviewers": ["@deads2k", "@lavalamp", "@liggitt", "@erictune", "@mbohlool", "@apelisse"],
+    "approvers": ["@deads2k", "@lavalamp"],
+    "editor": "@sttts",
+    "creationDate": "2018-07-31",
+    "lastUpdated": "2019-04-30",
+    "status": "implemented",
+    "seeAlso": [
+      "https://github.com/kubernetes/enhancements/pull/1002",
+      "/keps/sig-api-machinery/20180415-crds-to-ga.md"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Pruning for Custom Resources\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Excluding values from Pruning](#excluding-values-from-pruning)\n  - [Examples](#examples)\n  - [Opt-in and Opt-out of Pruning on CRD Level](#opt-in-and-opt-out-of-pruning-on-crd-level)\n  - [References](#references)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Alternatives Considered](#alternatives-considered)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nCustomResources store arbitrary JSON data without following the typical Kubernetes API  behaviour to prune unknown fields. This makes CRDs different, but also leads to security and general data consistency concerns because it is unclear what is actually stored in etcd. \n\nThis KEP proposes to add pruning of all fields which are not specified in the OpenAPI validation schemas given in the CRD. \n\nPruning will be opt-in in v1beta1 of `apiextensions.k8s.io` via\n\n```yaml\napiVersion: apiextensions.k8s.io/v1beta1\nkind: CustomResourceDefinition\nspec:\n  preserveUnknownFields: false\n  ...\n```\n\ni.e. CRDs created in v1beta1 default to disabled pruning. \n\n**Pruning will be enabled for every CRD created in v1** and we will hide `preserveUnknownFields` in v1 in that case.\n\nPruning can be disabled for subtrees of CustomResources by setting the `x-kubernetes-preserve-unknown-fields: true` vendor extension. This allows to store arbitrary JSON or RawExtensions. This will even be possible at the root level, even in v1. Adding that in every to each object in a schema leads to the old behaviour.\n\nPruning requires _structural schemas_ (as described in [KEP Vanilla OpenAPI Subset: Structural Schema](https://github.com/kubernetes/enhancements/pull/1002)) for all defined versions. Validation will reject the CRD with `preserveUnknownFields: false` otherwise.\n\n## Motivation\n\n* Native Golang based resources do pruning as a consequence of the JSON unmarshalling algorithm. This is has become a fundamental behaviour of Kubernetes API semantics that CustomResources break.\n* Pruning enforces consistency of data stored in etcd. Objects cannot suddenly render unaccessible because unexpected data breaks decoding.\n* Even if unexpected data in etcd is of the right type and does not break decoding, it has not gone through validation, and probably an admission webhook either does not exist for many CRDs or it won't have implemented pruning behaviour. Pruning at the decoding step enforces this (scenario: applying a new CR instance with new fields against a cluster with an old CRD manifest).\n* Pruning is a counter-measure to security attacks which make use of knowledge of future versions of APIs with new security relevant fields. Without pruning an attacker can prepare CustomResources with privileged fields set. On version upgrade of the cluster, these fields can suddenly become alive and lead to unallowed behaviour.\n\n### Goals\n\n* Prune unknown fields from CustomResources silently to match native type behaviour. Unknown means not specified in the OpenAPI validation spec.\n* Allow to opt-out of pruning via the OpenAPI validation spec for a whole subtree of  JSON objects.\n\n### Non-Goals\n* Add a strict mode to the REST API which rejects objects with unknown fields.\n\n## Proposal\n\nWe assume the CRD has _structural schemas_ (as defined in [KEP Vanilla OpenAPI Subset: Structural Schema](https://github.com/kubernetes/enhancements/pull/1002)).\n\nWe propose to\n\n1. derive the value-validation-less variant of the structural schema (trivial by definition of _structural schema_) and \n2. recursively follow the given CustomResource instance and the structural schema, removing fields from the former if they are not specified in the `properties` of the latter. Skip field removal in that recursion step (not for children) if `additionalProperties` is defined in the schema.\n3. return a deserialization error if the CustomResource instance JSON value and the type in the structural schema do not match\n4. fields of `metav1.TypeMeta` (`apiVersion` and `kind`) and `metav1.ObjectMeta` at the object root are implicitly specified. `metav1.ObjectMeta` is always pruned.\n\nNote that in (2) `additionalProperties: \u003cany-schema\u003e` stops field removal on that level. This includes `additionalProperties: false`, compare examples (4) and (5).\n\nWe do this in the serializer just after the binary payload has been unmarshalled into an `map[string]interface{}` for the request body and when reading from etcd, compare the yellow boxes in the following figure:\n\n![Decoding steps which must prune](20180731-crd-pruning-decoding.png)\n\nWe will also prune after mutating webhooks have been run.\n\n### Excluding values from Pruning\n\nThere are cases where parts of an object are verbatim JSON, i.e. without any applied schema and especially without a complete specification which allows to apply pruning.\n\nThe vendor extension `x-kubernetes-preserve-unknown-fields: true` proposed in (as defined in the [KEP Vanilla OpenAPI Subset: Structural Schema](https://github.com/kubernetes/enhancements/pull/1002)) serves exactly this purpose, with the following semantics:\n\nThe whole JSON subtree `X` at the level of `x-kubernetes-preserve-unknown-fields: true` and below is excluded from pruning, with the following exceptions:\n1. pruning starts again at `Y` in the subtree of children of `X` if `Y` is specified by `properties`.\n2. pruning starts again in the `metadata` child of `Y` in the subtree of `X` if `Y` is specified by `x-kubernetes-embedded-resource: true`. \n\nWe do not allow `x-kubernetes-preserve-unknown-fields: false`.\n\nNote that (1) does not apply if `X` and  `Y` are the same, compare examples (7) and (8) below.\n\n### Examples\n\n1. unspecified\n\n   ```yaml\n   type: object\n   ```\n   \n   Everything is pruned, i.e.\n\n   ```json\n   {\n     \"foo\": 42,\n     \"json\": {\"bar\": 43}\n   }\n   ```\n   \n   is pruned to\n   \n   ```json\n   {}\n   ```\n   \n2. `properties` at top-level\n\n   ```yaml\n   type: object\n   properties:\n     foo:\n       type: object\n   ```\n   \n   Everything other than `foo` is pruned, i.e.\n\n   ```json\n   {\n     \"foo\": {\n       \"abc\": 42\n     },\n     \"json\": {\"bar\": 43}\n   }\n   ```\n   \n   is pruned to\n   \n   ```json\n   {\n     \"foo\": {}\n   }\n   ```\n\n3. `properties` at multiple levels\n\n   ```yaml\n   type: object\n   properties:\n     foo:\n       type: object\n       properties:\n         bar:\n           type: object\n   ```\n   \n   Everything other than `foo` and `bar` is pruned, i.e.\n\n   ```json\n   {\n     \"foo\": {\n       \"bar\": {\n         \"abc\": 42\n       },\n       \"def\": 43\n     },\n     \"json\": {\"ghi\": 44}\n   }\n   ```\n   \n   is pruned to\n   \n   ```json\n   {\n     \"foo\": {\n       \"bar\": {}\n     }\n   }\n   ```\n   \n4. `additionalProperties` with a non-empty schema\n\n   ```yaml\n   type: object\n   properties:\n     foo:\n       type: object\n       additionalProperties:\n         type: object\n   ```\n   \n   Everything directly inside of `foo` stays (it is considered a string map), but\n   objects further down are pruned again because they are unspecified, i.e.\n\n   ```json\n   {\n     \"foo\": {\n       \"abc\": {\"x\": 42},\n       \"def\": {\"y\": 43}\n     },\n     \"json\": {\"ghi\": 44}\n   }\n   ```\n   \n   is pruned to\n   \n   ```json\n   {\n     \"foo\": {\n       \"abc\": {},\n       \"def\": {}\n     }\n   }\n   ```\n\n5. `additionalProperties: false`\n\n   ```yaml\n   type: object\n   properties:\n     foo:\n       type: object\n       additionalProperties: false\n   ```\n   \n   Everything directly inside of `foo` stays (it is considered a string map), but\n   objects further down are pruned again because they are unspecified, i.e.\n\n   ```json\n   {\n     \"foo\": {\n       \"abc\": {\"x\": 42},\n       \"def\": {\"y\": 43}\n     },\n     \"json\": {\"ghi\": 44}\n   }\n   ```\n   \n   is pruned to\n   \n   ```json\n   {\n     \"foo\": {\n       \"abc\": {},\n       \"def\": {}\n     }\n   }\n   ```\n   but validation will fail. We consider the semantical meaning of `false`  as value validation, not structural.\n   \n6. arbitrary JSON\n\n   ```yaml\n   type: object\n   properties:\n     json:\n       x-kubernetes-preserve-unknown-fields: true\n       nullable: true\n   ```\n\n   Inside of `.json` nothing is pruned, i.e.\n   \n   ```json\n   {\n     \"foo\": 42,\n     \"json\": {\"bar\": 43}\n   }\n   ```\n   \n   is pruned to\n   \n   ```json\n   {\n      \"json\": {\"bar\": 43}\n   }\n   ```\n   \n7. JSON, but with properties at the same level\n\n   ```yaml\n   type: object\n   properties:\n     json:\n       type: object\n       x-kubernetes-preserve-unknown-fields: true\n       nullable: true\n       properties:\n         bar:\n           type: object\n   ```\n\n   Inside of `.json` nothing is pruned, including everything in `bar`, i.e.\n   \n   ```json\n   {\n     \"foo\": 42,\n     \"json\": {\n       \"bar\": {\n         \"abc\": 43\n       },\n       \"def\": 44\n     }\n   }\n   ```\n   \n   is pruned to\n   \n   ```json\n   {\n      \"json\": {\n        \"bar\": {\n          \"abc\": 43\n        },\n        \"def\": 44\n      }\n   }\n   ```\n   \n8. JSON, but with properties at the same **and lower** levels\n\n   ```yaml\n   type: object\n   properties:\n     json:\n       type: object\n       x-kubernetes-preserve-unknown-fields: true\n       nullable: true\n       properties:\n         bar:\n           type: object\n           properties:\n             inner:\n               type: integer\n   ```\n\n   The `properties` for `bar` \"resets\" pruning to normal behaviour, i.e.\n   \n   ```json\n   {\n     \"foo\": 42,\n     \"json\": {\n       \"bar\": {\n         \"inner\":43,\n         \"abc\": 44\n       },\n       \"def\": 45\n     }\n   }\n   ```\n   \n   is pruned to\n   \n   ```json\n   {\n      \"json\": {\n        \"bar\": {\n          \"inner\": 43\n        },\n        \"def\": 45\n      }\n   }\n   ```\n   \n9. `additionalProperties` within JSON\n\n   ```yaml\n   type: object\n   properties:\n     json:\n       type: object\n       x-kubernetes-preserve-unknown-fields: true\n       nullable: true\n       additionalProperties:\n         type: object\n   ```\n\n   The `additionalProperties` disables pruning **at its level**, `x-kubernetes-preserve-unknown-fields: true` already has the same effect. Inside of the additional properties `x-kubernetes-preserve-unknown-fields: true` keeps being effective, i.e.\n   \n   ```json\n   {\n     \"foo\": 42,\n     \"json\": {\n       \"bar\": {\n         \"inner\":43,\n         \"abc\": 44\n       },\n       \"def\": 45\n     }\n   }\n   ```\n   \n   is pruned to\n   \n   ```json\n   {\n     \"json\": {\n       \"bar\": {\n         \"inner\":43,\n         \"abc\": 44\n       },\n       \"def\": 45\n     }\n   }\n   ```   \n10. embedded resource\n\n   ```yaml\n   type: object\n   properties:\n     object:\n       type: object\n       nullable: true\n       x-kubernetes-embedded-resource: true\n       x-kubernetes-preserve-unknown-fields: true\n   ```\n   \n   Here, inside of `.object` nothing is pruned with the exception of unknown fields under `.object.metadata`, i.e.\n\n   ```json\n   {\n     \"foo\": 42,\n     \"object\": {\n       \"bar\": 43,\n       \"abc\": 44,\n       \"metadata\": {\n         \"name\": \"example\",\n         \"garbage\": 45\n       }\n     }\n   }\n   ```\n      \n   is pruned to\n      \n   ```json\n   {\n      \"object\": {\n        \"bar\": 43,\n        \"abc\": 44,\n        \"metadata\": {\n          \"name\": \"example\"\n        }\n      }\n   }\n   ```\n \n11. implicit `metav1.TypeMeta` and `metav1.ObjectMeta`\n\n   ```yaml\n   type: object\n   ```\n\n   Pruning takes place, but `apiVersion`, `kind`, `metadata` and known fields under `metadata` are preserved, i.e.\n   \n   ```json\n   {\n     \"apiVersion\": \"example/v1\",\n     \"kind\": \"Foo\",\n     \"metadata\": {\n       \"name\": \"example\",\n       \"garbage\": 43\n     },\n     \"foo\": 42\n   }\n   ```\n   \n   is pruned to\n   \n   ```json\n   {\n     \"apiVersion\": \"example/v1\",\n     \"kind\": \"Foo\",\n     \"metadata\": {\n       \"name\": \"example\"\n     }\n   }\n   ```\n\n### Opt-in and Opt-out of Pruning on CRD Level\n\nWe will add a `preserveUnknownFields` flag to `CustomResourceDefinitionSpec` of `apiextensions.k8s.io/v1beta1` (and later v1):\n\n```go\ntype CustomResourceDefinitionSpec struct {\n  ...\n\t\n  // PreserveUnknownFields disables pruning of object fields which are not\n  // specified in the OpenAPI schema. apiVersion, kind, metadata and known\n  // fields inside metadata are excluded from pruning.\n  // Defaults to true in v1beta1, and will default to false in v1. \n  // Setting this field to false is considered an beta API.\n  PreserveUnknownFields *bool\n}\n```\n\nI.e. for `apiextensions.k8s.io/v1beta1` this will default to true for backwards compatibility.\n\nFor `apiextensions.k8s.io/v1` we will change the default to false and forbid true during creation and updates if it has been false before. In v1 the only way to opt-out from pruning is via setting `x-kubernetes-preserve-unknown-fields: true` in the schema.\n\nWe will hide `preserveUnknownFields` in v1 objects if it is not true.\n\nWhen CRD authors switch on pruning for an existing CRD, they are supposed to make their users trigger a data migration of existing objects in etcd, be it via an external migration mechanism, an operator rewriting all objects or manual procedures. \n\n### References\n\n* Old pruning implementation PR https://github.com/kubernetes/kubernetes/pull/64558, to be adapted \n* [OpenAPI v3 specification](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.0.md)\n* [JSON Schema](http://json-schema.org/)\n\n### Test Plan\n\n**blockers for alpha:**\n\nWe default `preserveUnknownFields` to true and hence switch off the whole code path doing pruning. This reduces risk for everybody not using this feature.\n\n* we add unit tests for the general pruning algorithm\n* we add apiextensions-apiserver integration tests to\n  * verify that the pruning feature is actually off if `preserveUnknownFields` is true.\n  * verify that `preserveUnknownFields` is defaulted to true.\n  * verify that pruning happens if `preserveUnknownFields` is false, for all versions in the CRD according to the schema of the respective version.\n  * verify that `metadata`, `apiVersion`, `kind` are preserved if `preserveUnknownFields` is false and there is no schema given in the CRD.\n  \n**blockers for beta:**\n\n* we implement and verify that `x-kubernetes-embedded-resource` and `x-kubernetes-preserve-unknown-fields` work as expected.\n* we add apiextensions-apiserver integration tests to\n  * verify that pruning happens on incoming request payloads, on read from storage and after calling mutating admission webhooks.\n\n**blockers for GA:**\n\n* we verified that performance of pruning is adequate and not considerably reducing throughput.\n\n### Graduation Criteria\n\n* the test plan is fully implemented for the respective quality level\n\n### Upgrade / Downgrade Strategy\n\nWe aim at implementing this feature right away as beta:\n\n* in order to get users' exposure to the feature with real CustomResourceDefinitions\n* because the API surface is tiny such that we don't expect change in that area\n* the pruning algorithm is simple enough that we feel confident with thourough test coverage that the risk is small.\n\nHence, we assume to be at beta in 1.15 and GA in 1.16 guided by the graduation criteria, leading the following upgrade/downgrade strategy:\n\n* setting `preserveUnknownFields` to false is considered beta quality in 1.15.\n* downgrading to 1.14 will lose `preserveUnknownFields: false`, but that's acceptable for beta.\n* downgrading from 1.16 (where pruning might be GA) to 1.15 will keep the same behaviour as we don't feature gate `preserveUnknownFields: false`.\n* upgrading from 1.14 will default to `preserveUnknownFields: true` and hence changes no behaviour.\n* upgrading from 1.15 will keep the value and hence change no behaviour.\n* when v1 of `apiextensions.k8s.io` is added, we will keep the old pruning behaviour for CRDs created in v1beta1 with `preserveUnknownFields: true`, but forbid `preserveUnknownFields: true` for every newly create v1 CRD. Hence, we keep backwards compatibility. \n  \n  Technically, it is still possible to get the old behaviour even in v1 by setting `x-kubernetes-preserve-unknown-fields: true` at the root level and in each `properties` statement. But we enforce the definition of a schema, at least with this minimal contents.\n\n### Version Skew Strategy\n\n* kubectl is not aware of pruning in relevant way\n* posting `preserveUnknownFields: false` beta quality CRDs to an old server will disable pruning. But that's acceptable.\n\n## Alternatives Considered\n\n* in [GDoc which preceded this KEP](https://docs.google.com/document/d/1rBn6SZM7NsWxzBN41J2kO2Odf07PeGPygatM_1RwofY/edit#heading=h.4qdisqud6z3t) we considered a number of alternatives, including using a skeleton schema approach. We decided against that because of its complex semantics. In contrast, the _structural schema_ of the [KEP Vanilla OpenAPI Subset: Structural Schema](https://github.com/kubernetes/enhancements/pull/1002) is the natural output of schema generators deriving a schema from Golang structs. This matches the behavior of pruning through JSON unmarshalling, independently of any value validation the developer adds on top.\n* we could allow nested `x-kubernetes-preserve-unknown-fields: false`, i.e. to switch on pruning again for a subtree. This might encourage non-Kubernetes-like API types. It is unclear whether there are use-cases we want to support which need this. We can add this in the future.\n* we could allow per-version opt-in/out of pruning via `preserveUnknownFields` in `CustomResourceDefinitionVersion`. For the goal of data consistency and security a CRD with semi-enabled pruning does not make much sense. The main reason to not enable pruning will probably be the lack of a complete structural schema. If this is added for one version, it should be possible for all other versions as well as it is less a technical, but a CRD development life-cycle question.\n* we intensively considered avoiding a new `x-kubernetes-preserve-unknown-fields` vendor extension in favor of recursive `additionalProperties` semantics. We decided against because:\n  \n  None of OpenAPI v3 schema constructs have effects recursively. We would conflict with that pattern.\n    \n    E.g. `additionalProperties: false` invalidates unknown fields only at its level in OpenAPI v3, for example:\n    ```yaml\n    type: object\n    additionalProperties: false\n    properties:\n      foo: {}\n    ```\n    (note: this is not allowed in CRDs, but in OpenAPI v3) forbids `{\"foo\":{},\"abc\":42}`, but not `{\"foo\":{\"abc\":42}}`. A recursive interpretation for pruning would diverge from this pattern.\n    \n    Another example:\n    ```yaml\n    additionalProperties:\n      minimum: 42\n    ```\n    forbids `{\"foo\":10}`, but not `{\"foo\":{\"bar\":10}}`.\n    \n* we stop pruning even for `additionalProperties: false` or any other additional properties schema. We considered to prune for `false`, but not for `true`. We decided against because:\n  \n  * it is unclear what should happen with pruning for non-empty schemas between `false` and `true`.\n  * it is infeasible to compute whether an arbitrary schema is empty (and hence equivalent to `false`) or not. Semantically empty schemas and `false` should behave the same behaviour. This includes pruning.\n  \n  By not pruning for any explicit value of `additionalProperties` (including `false`) we follow our principle of not trying to consider full semantics of OpenAPI including value validation when doing structural operations like pruning.\n  \n  Compare example (5): CustomResource validation will eventually forbid unpruned values. The semantical meaning of the schema `false` is considered non-structural and therefore not relevant for pruning.\n\n## Implementation History\n"
+  },
+  {
+    "id": "d3eaea735951e23b5e5cdbcfbdde2097",
+    "title": "legacyflags",
+    "authors": ["@mtaufen"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["sig-architecture", "sig-cluster-lifecycle", "wg-component-standard"],
+    "reviewers": ["@kubernetes/wg-component-standard"],
+    "approvers": ["@luxas", "@sttts"],
+    "editor": "TBD",
+    "creationDate": "2019-01-29",
+    "lastUpdated": "2019-04-02",
+    "status": "provisional",
+    "seeAlso": ["KEP-32"],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# kflag\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Requirements](#requirements)\n    - [General](#general)\n    - [Flag Registration](#flag-registration)\n    - [Flag Parsing](#flag-parsing)\n    - [Application of Parsed Flag Values](#application-of-parsed-flag-values)\n  - [Implementation Details](#implementation-details)\n    - [Code Location](#code-location)\n  - [Risks and Mitigations](#risks-and-mitigations)\n    - [Maintenance](#maintenance)\n    - [Migrating existing components](#migrating-existing-components)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nCreate a _legacyflags_ package, which wraps pflag and addresses the pain-points of backwards-compatible\nComponentConfig migration, and selective registration of third-party library flags.\n\n\n## Motivation\n\nMake it easier for component owners to implement\n[Versioned Component Configuration Files](https://docs.google.com/document/d/1FdaEJUEh091qf5B98HM6_8MS764iXrxxigNIdwHYW9c/edit#)\n(a.k.a. ComponentConfig).\n\nWhile a number of problems with existing machinery are highlighted in the above doc, this\nproposal is focused on the following two use-cases:\n* Enforcing [flag precedence](https://github.com/kubernetes/kubernetes/issues/56171) between values\n  specified on the command line and values from a config file, which is required to incrementally\n  migrate flags to a config file with backwards compatibility.\n* Preventing third-party code from\n  [implicitly registering flags](https://github.com/kubernetes/kubernetes/pull/57613)\n  against a component's command-line interface, which is required for components to maintain\n  explicit control over their command-line interface.\n\nRather than require other components to copy and customize the Kubelet's relatively complex\nsolutions to these problems, we should put the common functionality in a library, legacyflags, to reduce\nthe burden on component owners.\n\n### Goals\n\nProvide core Kubernetes components with common building blocks that simplify the following:\n* Application of parsed flag values to arbitrary structs (e.g. config structs).\n* Selective inclusion of globally-registered flags in third-party code.\n\n### Non-Goals\n\n* This KEP is not concerned with fixing various problems with Cobra, such as third-party flags\n  leaking into the default usage or help text. We may write a future KEP to cover these issues and\n  command building in general.\n* This KEP is not concerned with the structure of flag and configuration types or their composition\n  into aggregates. We may write a future KEP to cover this.\n* This KEP is not concerned with loading and parsing config files, or resolving references contained\n  in those files (e.g. relative paths).\n* This KEP is not concerned with dynamically configuring components, or otherwise managing\n  component lifecycle.\n* This KEP is not concerned with flag or config validation.\n* This KEP is not an attempt to fork pflag.\n\n## Proposal\n\nThis proposal recommends writing a wrapper package around the\n[pflag](https://godoc.org/github.com/spf13/pflag) library.\nThis wrapper package is called _legacyflags_ in this proposal.\nThe intent is _not_ to fork pflag.\n\nThis KEP is mostly about pinning down requirements and is accompanied by an example PR, linked\nin the below [Implementation Details](#implementation-details) section, which demonstrates _one_ way\nof accomplishing these goals.\n\n### Requirements\n\n#### General\n* Components that consume legacyflags should require less maintenance than components that implement\n  ComponentConfig without legacyflags.\n* legacyflags should be as compile-time type-safe as possible. There are many possible combinations of\n  config, so if runtime casts from interface{} are required, it may be difficult to test all\n  paths for safety.\n* Unless a component needs to access the underlying pflag FlagSet to implement additional custom\n  behavior, it should not need to use pflag directly.\n\n#### Flag Registration\n* Flags registered by third-party libraries must not appear in a component's command-line interface\n  unless the component owners explicitly allow it.\n* legacyflags should provide helpers for importing globally-registered third-party flags into a FlagSet.\n\n#### Flag Parsing\n* Components should only have to parse their command-line once, even if they have to apply the flag\n  values multiple times, or to multiple structs. The Kubelet's approach to enforcing flag-precedence\n  involves re-parsing the command-line, which led to significant complexity. This is especially\n  important if components allow dynamic config reloading at runtime, in which case flags need\n  to be applied on top of each new config.\n\n#### Application of Parsed Flag Values\n* Components must be able to apply the parsed values an arbitrary number of times without\n  accumulating side-effects. Components may apply values once for at least as many config sources\n  they can consume (at least once to access the initial flag values, and afterward to enforce flag\n  precedence on each config). Each stage of loading configuration (flags, config file, dynamic\n  config) is a potential decision point, prior to which flag precedence must be enforced.\n* Components should be able to apply the parsed flag values to structs that only represent a subset\n  of all flag values. For instance, if a component has a Flags struct and a Config struct,\n  it may need to apply values to each independently.\n* Components must be able to specify custom behavior for merging values during application.\n  Feature gates, for example, are merged piecemeal between the command-line and config files, with\n  gates specified on the command-line taking precedence.\n* Application of flag values should only consider the flags specified on the command line. It should\n  be careful, for example, to not overwrite values loaded from config files with the default values\n  of omitted flags.\n\n\n### Implementation Details\n\nFundamentally, separating parsing from application requires that we have a scratch space to parse\nthe initial values into, and a way to map that scratch space onto target objects.\n\nTo maintain type safety, this proposal recommends a two-layered approach, where legacyflags handles the\ncommon types, and components handle the component-specific types:\n* legacyflags Implements:\n  * Statically typed flag registration helpers that allocate scratch space.\n  * Statically typed helpers to apply the values of parsed flags to arbitrary targets.\n* Components Implement:\n  * Aggregate flag registration functions, which use legacyflags helpers to register groups of flags the\n    component cares about.\n  * Aggregate flag application functions, which use legacyflags helpers to apply a subset of flag values\n    to an arbitrary structure determined by the component.\n\nThis proposal recommends the approach in this example PR:\n* https://github.com/kubernetes/kubernetes/pull/73494\n\nSee also @sttts's prototype for an alternative approach that predates this KEP:\n* https://github.com/kubernetes/kubernetes/pull/72037\n\n#### Code Location\n\nWe will implement legacyflags in a new repo: `k8s.io/legacyflags`.\n\nDuring the implementation, we will maintain an open PR against `k8s.io/kubernetes` that vendors\nthe latest version of the library and demonstrates its use in a real component.\n\n### Risks and Mitigations\n\n#### Maintenance\nThe legacyflags package is a wrapper around an existing package, pflag.\nThe pflag package has a stable API, so we would at most expect to extend legacyflags when pflag is\nextended, but only in the event that we really need that extension. We expect this to be infrequent.\nGiven that legacyflags should make components easier to maintain (it reduces the complexity of the\nbootstrap process by making the flag precedence implementation easier to use and reason about),\nthe maintenance of legacyflags itself is likely time well spent.\n\n#### Migrating existing components\nThis should amount to an internal refactoring for existing components, with no external changes in\nbehavior (if we do change external-facing behavior, it's a bug). Some components may have better\ntest coverage than others, or unique features (like dynamic Kubelet config), so we will have to be\ncareful to test properly on a component-by-component basis as we migrate components to use legacyflags.\n\n## Graduation Criteria\n\n* Functional completeness:\n  * legacyflags implements helpers for the same set of types as pflag (or at least all the types used\n    by core Kubernetes components).\n  * We are satisfied that the example conversion PR demonstrates legacyflags's utility, measured by\n    whether the conversion PR can be merged.\n  * (optional) legacyflags absorbs the helpers that currently exist in `k8s.io/apiserver/pkg/util/flag`,\n    such as `MapStringBool`.\n* Migration completeness:\n  * All core Kubernetes components (excluding kubectl) use legacyflags to implement their\n    legacy command-line interface as they transition to ComponentConfig.\n\n## Implementation History\n\n* 2019-01-29: Initial KEP PR.\n"
+  },
+  {
+    "id": "a7e044f67c768f9bd744dc423210fb7f",
+    "title": "Watch Bookmark",
+    "authors": ["@wojtek-t"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["sig-scalability"],
+    "reviewers": ["@jpbetz"],
+    "approvers": ["@deads2k", "@lavalamp"],
+    "editor": "",
+    "creationDate": "2019-02-06",
+    "lastUpdated": "2019-04-30",
+    "status": "implemented",
+    "seeAlso": ["https://github.com/kubernetes/kubernetes/issues/73585"],
+    "replaces": ["n/a"],
+    "supersededBy": ["n/a"],
+    "markdown": "\n# Watch bookmark\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Risks and Mitigations](#risks-and-mitigations)\n  - [Test Plan](#test-plan)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Rejected alternatives](#rejected-alternatives)\n  - [Cache in kube-apiserver](#cache-in-kube-apiserver)\n  - [API for send bookmark](#api-for-send-bookmark)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nWatch API is one of the fundaments of Kubernetes API. The recommended pattern\nfor using watch API is to retrieve a collection of resources using consistent\nlist and then initiate a watch starting from a resourceVersion returned by the\nlist operation. If the client watch is disconnected, a new one can be restarted\nfrom the last returned resourceVersion.\n\nThis proposal make restarting watches cheaper from kube-apiserver performance\nperspective.\n\n## Motivation\n\nWhile running different scalability tests we observed that restarting watches\nmay cause significant load on kube-apiserver when watcher is observing a small\npercentage of changes (due to field or label selector). In extreme cases,\nreestablishing such watcher may even lead to falling out of history window\nand \"resource version too old\" errors (that requires full relist for that\nwatcher).\n\nThe reason for that is the fact that even if the last item received by watcher\nhas resourceVersion rv1, we may already know that there aren't any changes\na given watcher is interested in up to rv2 (rv1 \u003c rv2), but we don't have any\nway of communicating it to the watcher. As a result, when restarting a watch,\nclient again sends rv1 as a starting point, and we process all events with\nresourceVersion between rv1 and rv2 again unnecessarily.\n\nThe proposal presents a proper solution for that problem.\n\n### Goals\n\n- Reduce load on apiserver by minimizing amount of unnecessary watch events\nthat need to be processed after restarting a watch.\n- Reduce amount of undesired \"resource version too old\" errors on reestablishing\na watch.\n\n### Non-Goals\n\nThe following are nice-to-haves, but not primary goals:\n\n- Improve overall watch throughput and/or latency.\n\n## Proposal\n\nWe propose introducing a new type of watch event called `Bookmark`. With that\nchange, the possible watch event types will be:\n```\n  Added    EventType = \"ADDED\"\n  Modified EventType = \"MODIFIED\"\n  Deleted  EventType = \"DELETED\"\n  Error    EventType = \"ERROR\"\n  Bookmark EventType = \"BOOKMARK\"\n```\n\nWatch event with type Bookmark will represent information that all the objects\nup to a given resourceVersion has been processed for a given watcher. So even\nif the last event of other types contained object with resourceVersion rv1,\nreceiving a bookmark with resourceVersion rv2 means that there aren't\nany interesting objects for that watcher in between.\n\nGiven that we don't want to wait for v2 version of watch API with that change,\nan obvious requirement for introducing it is backward compatibility. Currently\nwatch Event type looks as following:\n```\n  type Event struct {\n    Type EventType\n    Object runtime.Object\n  }\n```\n\nAs a result, we will represent bookmark event by setting Bookmark type and\nObject of appropriate type with just ObjectMeta.ResourceVersion field set.\n\nUnfortunately, such a change would break existing clients, as an example\n[official decoder][]. As a result, we will extend `ListOptions` (which is\nhow we pass options to watch) with a boolean field where user can opt-in\nfor watch bookmarks:\n```\n  type ListOptions struct {\n    ...\n    AllowWatchBookmarks bool\n  }\n```\nWe consciously make it just a boolean flag - this gives kube-apiserver an\nability to choose when they should be send without setting any expectations\non user side how frequently and when it would be happening. In particular\nclient isn't guaranteed to get any bookmarks.\nSuch addition to `ListOptions` is also safe from backward-compatibility\npoint of view, old kube-apiserver will simply drop this field if set.\n\nOnce the API is extended, we add a support for sending bookmarks to watchcache.\nThe exact policy of sending them is to be determined, the ideas include:\n- send a bookmark if there weren't any event send to user in the last X\nseconds\n- if we know we will be closing (e.g. due to timeout) a watch, try to send\na bookmark immediately before that\n\nFinalizing the decision shouldn't block the initial version of this KEP.\n\n[official decoder]: https://github.com/kubernetes/kubernetes/blob/5d4795e14e02ac29273009d86ba3c5012684d5f4/staging/src/k8s.io/client-go/rest/watch/decoder.go#L57\n\n\n### Risks and Mitigations\n\nSending \"watch bookmarks\" may break clients not understanding them.\nAs a result, we make them explicitly opt-in.\n\n### Test Plan\n\nFor `Alpha` a set of unit tests will be added to verify that bookmarks are\nsend as assumed by the implementation.\nGiven the nature of the feature (no guarantees that bookmarks will be send\nto the watcher), no e2e tests will be added.\n\nFor `Beta` a metrics exposing number of processed `init events` is gathered\nby our scalability test framework so that the impact can be clearly proved\non our dashboards.\n\n## Graduation Criteria\n\nBeta:\n- Proved scalability/performance gain. With a simple POC I was able to\nreduce amount of processed \"init events\" by ~40x. So setting the minimum\ngoal on 10x without adding any additional visible overhead.\n- Generated informers make use of this new API.\n\nGA:\n- Enabled by default in Kubelet for watching pods for a release.\n- No complaints about the API for a release\n\n## Implementation History\n\n- 2019-02-12: KEP Summary, Moativation and Proposal merged\n- 2019-03-27: API changes approved in API review\n- 2019-04-16: Implementation merged\n- v1.15: Launched in `Alpha`\n\n## Rejected alternatives\n\nThe most important considered alternatives are mentioned below.\n\n### Cache in kube-apiserver\n\nInstead of introducing an API for bookmarks, we can try memorizing in watchcache\nwhat we already processed for a watcher and when it is restarted use that\ninformation. However, that would require being able to identify and match a\nwatcher across restarts which is non-trivial. Moreover, it doesn't work in\nHA setups with multiple kube-apiserver.\n\n### API for send bookmark\n\nThis is similar to what was done an etcd, where an API was added to notify\nall watchers about current resourceVersion. However, such API would be hard\nto manage in kube-apiserver, and we don't really want to notify everyone at\nthe same time.\n"
+  },
+  {
+    "id": "50dbfc8c0f926b67d20d82ca1bc6c360",
+    "title": "API Server Network Proxy",
+    "authors": ["@cheftako", "@anfernee"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["sig-network", "sig-cloud-provider"],
+    "reviewers": [
+      "TBD",
+      "@lavalamp",
+      "@deads2k",
+      "@bowei",
+      "@andrewsykim",
+      "@justinsb",
+      "@krousey",
+      "@khenidak",
+      "@mikedanese"
+    ],
+    "approvers": [
+      "@deads2k - For Kube API Server portion of KEP",
+      "@bowei - For networking/proxy portion of KEP"
+    ],
+    "editor": "@calebamiles",
+    "creationDate": "2019-02-25",
+    "lastUpdated": "2019-04-30",
+    "status": "implementable",
+    "seeAlso": [
+      "https://goo.gl/qiARUK - Network Proxy design proposal",
+      "https://goo.gl/ipwDkX - Explicit API server to node communications",
+      "https://github.com/kubernetes-sigs/apiserver-network-proxy - Reference implementations of API Server Network Proxy"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# API Server Network Proxy\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Definitions](#definitions)\n- [Proposal](#proposal)\n  - [Network Context](#network-context)\n  - [Proxy gRPC definition](#proxy-grpc-definition)\n  - [Konnectivity Server](#konnectivity-server)\n  - [Direct Connection](#direct-connection)\n  - [Kubernetes API Server Outbound Requests](#kubernetes-api-server-outbound-requests)\n  - [Testing the Solution](#testing-the-solution)\n  - [Security](#security)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n- [User Stories](#user-stories)\n    - [Combined Master and Node Network](#combined-master-and-node-network)\n    - [Master and Untrusted Node Network](#master-and-untrusted-node-network)\n    - [Master and Node Networks which are not IP Routable](#master-and-node-networks-which-are-not-ip-routable)\n    - [Better Monitoring](#better-monitoring)\n- [Design Details](#design-details)\n  - [Risks and Mitigations](#risks-and-mitigations)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Alternatives [optional]](#alternatives-optional)\n- [Infrastructure Needed [optional]](#infrastructure-needed-optional)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nWe will build an extensible system which controls network traffic from the Kube API Server.\nWe will add a traffic egress or network proxy system. The KAS can be configured to send traffic\n(or not) to one or more of the proxies. Users can drop in custom proxies if the\ndefault behavior is insufficient.\n\n## Motivation\n\nKubernetes has outgrown the [SSH tunnels](https://github.com/kubernetes/kubernetes/issues/54076).\nThey complicate KAS code and only one cloud provider implemented them.\nAfter a year of deprecation time, they will be removed in an upcoming release.\n\nIn retrospect, having an explicit level of indirection that separates user-initiated network traffic from API\nserver-initiated traffic is a useful concept.\nCloud providers want to control how API server to pod, node and service network traffic is implemented.\nCloud providers may choose to run their API server (control network) and the cluster nodes (cluster network)\non isolated networks. The control and cluster networks may have overlapping IP addresses.\nThere for they require a non IP routing layer (SSH tunnel are an example).\nAdding this layer enables metadata audit logging. It allows validation of outgoing API server connections.\nStructuring the API server in this way is a forcing function for keeping architectural layering violations out of apiserver.\nIn combination with a firewall, this separation of networks protects against security concerns such as\n[Security Impact of Kubernetes API server external IP address proxying](https://groups.google.com/d/msg/kubernetes-security-announce/tyd-MVR-tY4/tyREP9-qAwAJ).\n\n### Goals\n\nDelete the SSH Tunnel/Node Dialer code from Kube APIServer.\nEnable admins to fix https://groups.google.com/d/msg/kubernetes-security-announce/tyd-MVR-tY4/tyREP9-qAwAJ.\nAllow isolation of the Control network from the Cluster network.\n\n### Non-Goals\n\nBuild a general purpose Proxy which does everything. (Users should build their own\ncustom proxies with the desired behavior, based on the provided proxy)\nHandle requests from the Cluster to the Control Plane. (The proxy can be extended to\ndo this. However that is left to the User if they want that behavior)\n\n## Definitions\n\n- **Master Network** An IP reachable network space which contains the master components, such as Kubernetes API Server,\nConnectivity Proxy and ETCD server.\n- **Node Network** An IP reachable network space which contains all the clusters Nodes, for alpha.\nWorth noting that the Node Network may be completely disjoint from the Master network.\nIt may have overlapping IP addresses to the Master Network or other means of network isolation.\nDirect IP routability between cluster and master networks should not be assumed.\nLater version may relax the all node requirement to some.\n- **KAS** Kubernetes API Server, responsible for serving the Kubernetes API to clients.\n- **KMS** Key Management Service, plugins for secrets encryption key management\n- **Egress Selector** A component built into the KAS which provides a golang dialer for outgoing connection requests.\nThe dialer provided depends on NetworkContext information.\n- **Konnectivity Server** The proxy server which runs in the master network.\nIt has a secure channel established to the cluster network.\nIt could work on either a HTTP Connect mechanism or gRPC.\nIf the former it would exposes a gRPC interface to KAS to provide connectivity service.\nIf the latter it would use standard HTTP Connect.\nFormerly known the the Network Proxy Server.\n- **Konnectivity Agent** A proxy agent which runs in the node network for\n  establishing the tunnel.\nFormerly known as the Network Proxy Agent.\n- **Flat Network** A network where traffic can be successfully routed using IP.\nImplies no overlapping (i.e. shared) IPs on the network.\n\n## Proposal\n\nWe will run a connectivity server inside the master network.\nIt could work on either a HTTP Connect mechanism or gRPC.\nFor the alpha version we will attempt to get this working with HTTP Connect.\nWe will evaluate HTTP Connect for scalability, error handling and traffic types.\nFor scalability we will be looking at the number of required open connections.\nIncreasing usage of webhooks means we need better than 1 request per connection (multiplexing).\nWe also need the tunnel to be tolerant of errors in the requests it is transporting.\nHTTP-Connect only supports HTTP requests and not things like DNS requests.\nWe assume that for HTTP URL request,s it will be the proxy which does the DNS lookup.\nHowever this means that we cannot have the KAS perform a DNS request to then do a follow on request.\nIf no issues are found with HTTP Connect in these areas we will proceed with it.\nIf an issue is found then we will update the KEP and switch the client to the gRPC solution.\nThis should be as simple as switching the connection mode of the client code.\n\nIt may be desirable to allow out of band data (metadata) to be transmitted from the KAS to the Proxy Server.\nWe expect to handle metadata in the HTTP Connect case using http 'X' headers on the Connect request.\nThis means that the metadata can only be sent when establishing a KAS to Proxy tunnel.\nFor the GRPC case we just update the interface to the KAS.\nIn this case the metadata can be sent even during tunnel usage.\n\nEach connectivity proxy allows secure connections to one or more cluster networks.\nAny network addressed by a connectivity proxy must be flat.\nCurrently the only mechanism for handling overlapping IP ranges in Kubernetes is the Proxy.\nNon IP routable traffic, past the proxy, would need to be a non Kubernetes mechanism to route.\n\nRunning the connectivity proxy in a separate process has a few advantages.\n- The connectivity proxy can be extended without recompiling the KAS.\nAdministrators can run their own variants of the connectivity proxy.\n- Traffic can be audited or forwarded (eg. via a proprietary VPN) using a custom connectivity proxy.\n- The separation removes master \u003c-\u003e cluster connectivity concerns from the KAS.\n- The code and responsibility separation lowers the complexity of the KAS code base.\n- The separation reduces the effects of issue such as crashes in the connectivity impacting the KAS.\nConnectivity issues will not stop the KAS from serving API requests.\nThis is important as serving API requests may be necessary in order to fix the crashes.\nA problem with a node, set of nodes or load-balancers configuration, may be fixed with API requests.\n\n![API Server Network Proxy Simple Cluster](NetworkProxySimpleCluster.png)\nThe diagram shows API Server’s outgoing traffic flow.\nThe user (in blue box), master network (in purple cloud) and\na cluster network (in green cloud) are represented.\n\nThe user (blue) initiates communication to the KAS.\nThe KAS then initiates connections to other components.\nIt could be node/pod/service in cluster networks (red dotted arrow to green cloud),\nor etcd for storage in the same master network (blue arrow) or mutate the request\nbased on an admission web-hook (red dotted arrow to purple cloud).\nThe KAS handles these cases based on NetworkContext based traffic routing.\nThe connectivity proxy should be able to do routing solely based on IP.\nThe proxy should not require the NetworkContext. This means the service CIDR,\nnode CIDR and pod CIDR of each cluster network cannot overlap.\n\n### Network Context\n\nThe minimal NetworkContext looks like the following struct in golang:\n\n```go\ntype EgressType int\n\nconst (\n    // Master is the EgressType for traffic intended to go to the control plane.\n    Master EgressType = iota\n    // Etcd is the EgressType for traffic intended to go to Kubernetes persistence store.\n    Etcd\n    // Cluster is the EgressType for traffic intended to go to the system being managed by Kubernetes.\n    Cluster\n)\n\n// NetworkContext is the struct used by Kubernetes API Server to indicate where it intends traffic to be sent.\ntype NetworkContext struct {\n    // EgressSelectionName is the unique name of the\n    // EgressSelectorConfiguration which determines\n    // the network we route the traffic to.\n    EgressSelectionName EgressType\n}\n```\n\nEgressSelectionName specifies the network to route traffic to.\nThe KAS starts with a list of registered konnectivity service names. These\ncorrespond to networks we route traffic to. So the KAS knows where to\nproxy the traffic to, otherwise it return an “Unknown network” error.\n\nThe KAS starts with a proxy configuration like the below example.\nThe example specifies 4 networks. \"direct\" specifies the KAS talking directly\non the local network (no proxy). \"master\" specifies the KAS talks to a proxy\nlistening at 1.2.3.4:5678. \"cluster\" specifies the KAS talk to a proxy\nlistening at 1.2.3.5:5679. While these are represented as resources\nthey are not intended to be loaded dynamically. The names are not case\nsensitive. The KAS loads this resource lsit as a configuration at start time.\n\n```yaml\napiVersion: apiserver.k8s.io/v1alpha1\nkind: EgressSelectorConfiguration\negressSelections:\n- name: direct\n  connection:\n    type: direct\n- name: master\n  connection:\n    type: grpc\n    url: grpc://1.2.3.4:5678\n    caBundle: file1.pem\n    clientKeyFile: proxy-client1.key\n    clientCertFile: proxy-client1.crt\n- name: cluster\n  connection:\n    type: grpc\n    url: grpc://1.2.3.5:5679\n    caBundle: file2.pem\n    clientKeyFile: proxy-client2.key\n    clientCertFile: proxy-client2.crt\n```\n\nNetworkContext could be extended to contain more contextual information.\nThis would allow smarter routing based on the k8s object KAS is processing\nor which user/tenant tries to initiate the request, etc.\n\n### Proxy gRPC definition\n\nIn order to serve a proxy request, one gRPC bidirectional stream on proxy\nserver is created to serve it. It's a 1:1 mapping from TCP connection to a\ngRPC stream, so the state of TCP connection is exactly the same as the gRPC\nstream state.\n\n```grpc\nsyntax = \"proto3\";\n\nservice ProxyService {\n  // Proxy a TCP connection to a remote address defined by ConnectParam.\n  // The ConnectParam is defined in metadata under key \"x-kube-net-proxy\".\n  // metadata[\"x-kube-net-proxy\"] = base64.Encode(proto.Marshal(connectOptions))\n  rpc Proxy(stream Payload) returns (stream Payload) {}\n}\n\n// ConnectOptions defines the remote TCP endpoint to connect\nmessage ConnectOptions {\n  string remote_addr = 1; // remote address to connect to. e.g. 8.8.8.8:53\n}\n\n// Payload defines a TCP payload.\nmessage Payload {\n  bytes data = 1;\n}\n```\n\n### Konnectivity Server\n\nThe Konnectivity Server (connectivity proxy(s)) can run in the same container as the KAS.\nIt should run on the same machine and must run in the same flat network as the KAS.\nIt listens on a port for gRPC connections from the KAS.\nThis port would be for forwarding traffic to the appropriate cluster.\nIt should have an administrative port speaking https.\nThe administrative port serves the liveness probe and metrics.\nThe liveness probe prevents a partially broken cluster\nwhere the KAS cannot connect to the cluster. This port also serves\npprof debug commands and monitoring data for the proxy.\n\n### Direct Connection\n\nThis connection type uses the default dialer.\nThis allows use of the connectivity service without the connectivity proxy.\nThis is a quick way to run the system in a “legacy” or fallback mode.\nSimple clusters (not needing network segregation) run this way to avoid the overhead\n(in latency or configuration) of the connectivity proxy.\n\n### Kubernetes API Server Outbound Requests\n\nThe majority of the KAS communication originates from incoming requests.\nHere we cover the outgoing requests. This is our understanding of those requests\nand some details as to how they fit in this model. For the alpha release we\nsupport 'master', 'etcd' and 'cluster' connectivity service names.\n\n- **ETCD** It is possible to make etcd talk via the proxy.\nThe etcd client takes a transport.\n(https://github.com/etcd-io/etcd/blob/master/client/client.go#L101)\nWe will add configuration as to which proxy an etcd client should use.\n(https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go)\nThis will add an extra process hop to our main scaling axis.\nWe will scale test the impact and publish the results. As a precaution\nwe will add an extra network configuration 'etcd' separate from ‘master’.\nEtcd requests can be configured separately from the rest of 'master'.\n- **Pods/Exec**, **Pods/Proxy**, **Pods/Portforward**, **Pods/Attach**, **Pods/Log**\nPod requests (and pod sub-resource requests) are meant for the cluster\nand will be routed based on the ‘cluster’ NetworkContext.\n- **Nodes/Proxy**\nNode requests (and node sub-resource requests) are meant for the cluster\nand will be routed based on the ‘cluster’ NetworkContext.\n- **Services/Proxy**\nService requests (and service sub-resource requests) are meant for the cluster\nand will be routed based on the ‘cluster’ NetworkContext.\n- **Admission Webhooks**\nAdmission webhooks can be destined for a service or a URL.\nIf destined for a service then the service rules apply (send to 'cluster').\nIf destined for a URL then we will use the ‘master’ NetworkContext.\n- **Aggregated API Server (and OpenAPI requests for aggregated resources)**\nAggregated API Servers can be destined for a service or a URL.\nIf destined for a service then the service rules apply.\nIf destined for a URL then we will use the ‘master’ NetworkContext.\n- **Authentication, Authorization and Audit Webhooks**\nThese Webhooks use a kube config file to determine destination.\nGiven that we use a ‘master’ NetworkContext.\n- **ImagePolicyWebhook**\nThe image policy webhook uses a kube config file to determine destination.\nGiven that we use a ‘master’ NetworkContext.\n- **KMS GRPC Service**\nKMS connects with an ‘endpoint’ (not the resource) via gRPC.\nThe service at the endpoint provides the secret information for use in encryption.\nThis is not a user space configurable system.\nGiven that we use a ‘master’ NetworkContext.\n\n### Testing the Solution\n\nWe will test using a network namespace to partition the KAS from the test nodes.\nIt is then impossible to connect directly from the KAS to the test nodes.\nThis ensures that the proxy must be used for logs, exec, port forward, aggregation and webhooks.\nWe run with this configuration and a direct configuration for these specific features.\nThis ensures that the solution works and will continue to work.\n\n### Security\n\nOne motivation for network proxy is providing a mechanism to secure\nhttps://groups.google.com/d/msg/kubernetes-security-announce/tyd-MVR-tY4/tyREP9-qAwAJ.\nThis, in conjunction with a firewall or other network isolation, fixes the security concern.\n\n### Implementation Details/Notes/Constraints\n\nYou may want to check the original design doc for alternatives and futures considered. https://goo.gl/qiARUK.\nPlease make sure you are a member of kubernetes-dev@googlegroups.com to view the doc.\nIt is also worth looking at https://github.com/kubernetes-sigs/apiserver-network-proxy as it contains the reference\nimplementation of the API Server Network Proxy.\n\n## User Stories\n\n#### Combined Master and Node Network\n\nCustomers can run a cluster which combines the master and cluster networks.\nThey configure all their connectivity configuration to direct.\nThis bypasses the proxy and optimizes the performance. For a customer with no\nsecurity concerns with combined network, this is a fairly simple straight forward configuration.\n\n#### Master and Untrusted Node Network\n\nA customer may want to isolate their master from their cluster network. This may be a\nsimple separation of concerns or due to something like running untrusted workloads on\nthe cluster network. Placing a firewall between the master and\ncluster networks accomplishes this. A few ports for the KAS public port and Proxy public port\nare opened between these networks. Separation of concerns minimizes the\naccidental interactions between the master and cluster networks. It minimizes bandwidth\nconsumption on the cluster network negatively impacting the control plane. The\ncombination of firewall and proxy minimizes the interaction between the networks to\na set which can be more easily reasoned about, checked and monitored.\n\n#### Master and Node Networks which are not IP Routable\n\nIf master and cluster network CIDRs are not controlled by the same entity, then they\ncan end up having conflicting IP CIDRs. Traffic cannot be routed between\nthem based strictly on IP address. The connection proxy solves this issue.\nIt also solves connectivity using a VPN tunnel. The proxy offloads the work off sending traffic\nto the cluster network from the KAS. The proxy gives us extensibility.\n\n#### Better Monitoring\n\nInstrumenting the network proxy requests with out of band data\n(Eg. requester identity/tradition context) enables a Proxy to\nprovide increased monitoring of Master originated requests.\n\n\n## Design Details\n\n### Risks and Mitigations\n\nThe primary risk of this solution would seem to be some portion of the proxy or agent failing.\nFor existing clusters which do not depend on SSH Tunnels or any of the new functionality, the\nmitigation would be to set all networks to direct. This should bypass the proxy and allow\nthe system to work as it does today. For anyone using SSH Tunnels we are planning to support\nboth for several releases.\n\n### Test Plan\n\nThe primary test plan is to set up a network namespace with a firewall dividing the master and cluster\nnetworks. Then running the existing tests for logs, proxy and portforward to ensure the\nrouting works correctly. It should work with the correct configuration and fail correctly\nwith a direct configuration. Normal tests would be run with the direct\nconfiguration to ensure the mitigation is working correctly.\n\n\nPlease adhere to the [Kubernetes testing guidelines][testing-guidelines] when drafting this test plan.\n\n[testing-guidelines]: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md\n\n### Graduation Criteria\n\nAlpha:\n\n- Feature is turned off in the KAS by default. Enabled by adding ConnectivityServiceConfiguration.\n- Kubernetes will not ship with a network proxy. The feature will work with the sample network proxy in https://github.com/kubernetes-sigs/apiserver-network-proxy\n- Demonstrate that the API Server Network Proxy eliminates the need for the SSH Tunnels.\n\nBeta:\n\n- All [Kube API Server egress points](#kubernetes-api-server-outbound-requests) have been implemented to use the\n  EgressSelector.\n- Have official releases of the [Konnectivity Server and Agent](https://github.com/kubernetes-sigs/apiserver-network-proxy) reference implementations.\n- Have at least one OSS kube-up implementation where the feature can be turned on and\n  demonstrated.\n- Have run a basic load test with egresses enabled through the Konnectivity\n  Server to demonstrate that concurrent requests work with Admission Webhooks.\n- Tests for EgressSelector.\n- e2e test with a functioning cluster with the EgressSelector conifgured to use\n  a KonnectivityService.\n- Add metrics and trace around the Egress Lookup/Dial code. Make sure we know\n  how many egresses of each type are returned. Make sure we know how long we\n  are spending dialing out.\n- Ensure we have metrics on each existing egress use case.\n\n## Implementation History\n\n- Feature went Alpha in 1.16 with limited functionality. It will cover the log\n  sub resource and communication to the etcd server.\n\n## Alternatives [optional]\n\n- Leave SSH Tunnels (deprecated) in the KAS. Prevents us from making the KAS cloud provider agnostic. Blocks out of tree effort.\n- Build equivalent functionality into the KAS. Is not extensible. Essentially has the same issues as SSH Tunnels.\n- Use a socks5 proxy. No standard mTLS mechanism for securing traffic. Does not actually act as a standard. More complicated implementation.\n\n## Infrastructure Needed [optional]\n\nAny one wishing to use this feature will need to create network proxy images/pods on the master and set up the ConnectivityServiceConfiguration.\nThe network proxy provided is meant as a reference implementation. Users as expected to extend it for their needs.\n"
+  },
+  {
+    "id": "00fd71ace1baabd6c24f00cc3c1d9e7e",
+    "title": "Priority and Fairness for API Server Requests",
+    "authors": ["@MikeSpreitzer", "@yue9944882"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["wg-multitenancy"],
+    "reviewers": ["@deads2k", "@lavalamp"],
+    "approvers": ["@deads2k", "@lavalamp"],
+    "editor": "TBD",
+    "creationDate": "2019-02-28",
+    "lastUpdated": "2019-02-28",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Priority and Fairness for API Server Requests\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n  - [Future Goals](#future-goals)\n- [Proposal](#proposal)\n  - [Request Categorization](#request-categorization)\n  - [Assignment to a Queue](#assignment-to-a-queue)\n    - [Queue Assignment Proof of Concept](#queue-assignment-proof-of-concept)\n    - [Probability of Collisions](#probability-of-collisions)\n  - [Resource Limits](#resource-limits)\n    - [Primary CPU and Memory Protection](#primary-cpu-and-memory-protection)\n    - [Secondary Memory Protection](#secondary-memory-protection)\n    - [Latency Protection](#latency-protection)\n  - [Queuing](#queuing)\n  - [Dispatching](#dispatching)\n    - [Fair Queuing for Server Requests](#fair-queuing-for-server-requests)\n  - [Example Configuration](#example-configuration)\n  - [Reaction to Configuration Changes](#reaction-to-configuration-changes)\n  - [Default Behavior](#default-behavior)\n  - [Prometheus Metrics](#prometheus-metrics)\n  - [Testing](#testing)\n  - [Observed Requests](#observed-requests)\n    - [Loopback](#loopback)\n    - [TokenReview from kube-controller-manager](#tokenreview-from-kube-controller-manager)\n    - [SubjectAccessReview by Aggregated API Server](#subjectaccessreview-by-aggregated-api-server)\n    - [GET of Custom Resource by Administrator Using Kubectl](#get-of-custom-resource-by-administrator-using-kubectl)\n    - [Node to Self](#node-to-self)\n    - [Other Leases](#other-leases)\n    - [Status Update From System Controller To System Object](#status-update-from-system-controller-to-system-object)\n    - [Etcd Operator](#etcd-operator)\n    - [Garbage Collectors](#garbage-collectors)\n    - [Kube-Scheduler](#kube-scheduler)\n    - [Kubelet Updates Pod Status](#kubelet-updates-pod-status)\n    - [Controller in Hosted Control Plane](#controller-in-hosted-control-plane)\n    - [LOG and EXEC on Workload Pod](#log-and-exec-on-workload-pod)\n    - [Requests Over Insecure Port](#requests-over-insecure-port)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [References](#references)\n  - [Design Considerations](#design-considerations)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Drawbacks](#drawbacks)\n- [Alternatives](#alternatives)\n- [Infrastructure Needed](#infrastructure-needed)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n**ACTION REQUIRED:** In order to merge code into a release, there must be an issue in [kubernetes/enhancements] referencing this KEP and targeting a release milestone **before [Enhancement Freeze](https://github.com/kubernetes/sig-release/tree/master/releases)\nof the targeted release**.\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e., [kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be updated for the enhancement to be released.\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nThis KEP generalizes the existing max-in-flight request handler in the\napiserver to make more distinctions among requests and provide\nprioritization and fairness among the categories of requests.  An\noutline of the request handling in an apiserver can be found at\nhttps://speakerdeck.com/sttts/kubernetes-api-codebase-tour?slide=18 .\n\n## Motivation\n\nToday the apiserver has a simple mechanism for protectimg itself\nagainst CPU and memory overloads: max-in-flight limits for mutating\nand for readonly requests.  Apart from the distinction between\nmutating and readonly, no other distinctions are made among requests;\nconsequently there can be undesirable scenarios where one subset of\nthe request load crowds out other parts of the request load.  Also,\nthe dispatching of these requests against two independent limits is\nnot\n[work-conserving](https://en.wikipedia.org/wiki/Work-conserving_scheduler).\n\n### Goals\n\nFollowing are some bad scenarios that can happen today and which\nshould be preventable when this KEP is in place.\n\n- Self-Maintenance crowded out.  Some requests are for system\n  self-maintenance, such as: node heartbeats, kubelet and kube-proxy\n  work on pods, services, secrets, etc involved in the system's\n  self-hosting, and leader elections for system controllers.  In an\n  overload scenario today there is no assurance of priority for these\n  self-maintenance requests.\n\n- Priority Inversions.  In the course of serving request A, there are\n  such as requests issued by an admission plugin (e.g., ResourceQuota)\n  or any client-ish code in a registry strategy.  Another example is\n  requests issued by an external server that itself is serving\n  call-outs from an apiserver (e.g., admission web-hooks).  Other\n  examples include requests from an aggregated apiserver to create\n  TokenReview and SubjectAccessReview objects.  Today it is possible\n  that the very load imposed by request A crowds out requests B\n  involved in serving A.\n\n- Guaranteed capacity for Low Priority.  There can be thundering herds\n  with higher priority running many minutes in the cluster. In order\n  to prevent an outage for the normal users connecting the cluster,\n  requests with higher priority will not completely starve out the\n  whole capacity.\n\n- Garbage Collector crowded out.  The garbage collector should keep up\n  with the workload, but in an overload situation today this is not\n  assured to happen.\n\n- Deployment of Doom.  We had a situation where a bug in the\n  Deployment controller caused it to run amuck under certain\n  circumstances, issuing requests in a tight loop.  We would like\n  controller bugs to not take the whole system down.\n\n- Kubelet Amuck.  The controller that runs amuck might not be a\n  central singleton, it could be a kubelet, kube-proxy, or other\n  per-node or otherwise multiplied controller.  In such a situtation\n  we would like only the guilty individual to suffer, not all its\n  peers and the rest of the system.\n\n- Overbearing or buggy tenants.  In a multi-tenant scenario, we would\n  like to prevent some tenants from crowding out the others.  Various\n  usage scenarios involve identifying the tenant in the following\n  ways.\n  \n  - Each tenant corresponds with a kube API namespace.\n\n  - Each tenant corresponds with a user name.\n\n  - Each tenant corresponds with a prefix of the user name.\n\n  - Each tenant corresponds with a user's group.  Other groups may\n    exist.  There is a subset of the groups that serve to identify\n    tenants.  Each user belongs to exactly one of the\n    tenant-identifying groups.\n\nThis KEP introduces new functionality in apiservers, and it should be\npossible to monitor this functionality through Prometheus metrics\navailable from the apiservers.\n\nThis KEP introduces new configuration objects, and they really\nmatter; it should be easy to apply suitable access controls.\n\nThere should be some reasonable defaults.\n\n### Non-Goals\n\nThis will be our first cut at a significant area of functionality, and\nour goals are deliberately modest.  We want to implement something\nuseful but limited and get some experience before going further.  Our\nprecise modesty has not been fully agreed.  Following is an initial\nstake in the ground.\n\n- No coordination between apiservers nor with a load balancer is\n  attempted.  In this KEP each apiserver independently protects\n  itself.  We imagine that later developments may add support for\n  informing load balancers about the load state of the apiservers.\n\n- The fairness does not have to be highly precise.  Any rough fairness\n  will be good enough.\n\n- WATCH and CONNECT requests are out of scope.  These are of a fairly\n  different nature than the others, and their management will be more\n  complex.  Also they are arguably less of an observed problem.\n\n- We are only concerned with protection of the CPU and memory of the\n  apiserver.  We are not concerned with etcd performance, nor output\n  network bandwidth, nor the ability of clients to consume output.\n\n- This KEP will not attempt auto-tuning the capacity limit(s).  Instead\n  the administrator will configure each apiserver's capacity limit(s),\n  analogously to how the max-in-flight limits are configured today.\n\n- This KEP will not attempt to reproduce the functionality of the\n  existing event rate limiting admission plugin.  Events are a\n  somewhat special case.  For now we intend to simply leave the\n  existing admission plugin in place.\n\n- This KEP will not attempt to protect against denial-of-service\n  attacks at lower levels in the stack; it is only about what can be\n  done at the identified point in the handler chain.\n\n- This KEP does not introduce threading of additional information\n  through webhooks and/or along other paths to support avoidance of\n  priority inversions.  While that is an attractive thing to consider\n  in the future, this KEP is deliberately limited in its ambition.\n  The intent for this KEP is to document that for the case of requests\n  that are secondary to some other requests the configuration should\n  identify those secondary requests and give them sufficiently high\n  priority to avoid priority inversion problems.  That will\n  necessarily be approximate, and we settle for that now.\n\n### Future Goals\n\nTo recap, there are some issues that we have decided not to address\nyet but we think may be interesting to consider in the future.\n\n- Helping load balancers do a better job, considering each apiserver's\n  current load state.\n\n- Do something about WATCH and/or CONNECT requests.\n\n- React somehow to etcd overloads.\n\n- Generate information to help something respond to downstream\n  congestion.\n\n- Auto-tune the resource limit(s) and/or request cost(s).\n\n- Be more useful for events.\n\n- Thread additional information along the paths needed to enable more\n  precisely targeted avoidance of priority inversions.\n\n\n## Proposal\n\nIn short, this proposal is about generalizing the existing\nmax-in-flight request handler in apiservers to add more discriminating\nhandling of requests.  The overall approach is that each request is\ncategorized to a priority level and a queue within that priority\nlevel; each priority level dispatches to its own isolated concurrency\npool; within each priority level queues compete with even fairness.\n\n### Request Categorization\n\nUpon arrival at the handler, a request is assigned to exactly one\n_priority level_ and exactly one _flow_ within that priority level.\nThis is done by matching the request against a configured set of\nFlowSchema objects.  This will pick exactly one best matching\nFlowSchema, and that FlowSchema will identify a RequestPriority object\nand the way to compute the request’s flow identifier.\n\nA RequestPriority object defines a priority level.  Each one is either\n_exempt_ or not.  There should be at most one exempt priority level.\nBeing exempt means that requests of that priority are not subject to\nconcurrency limits (and thus are never queued) and do not detract from\nthe concurrency available for non-exempt requests.  In a more\nsophisticated system, the exempt priority level would be the highest\npriority level.\n\nIt is expected that there will be only a few RequestPriority objects.\nIt is expected that there may be a few tens of FlowSchema objects.  At\none apiserver there may be tens of thousands of flow identifiers seen\nclose enough in time to have some interaction.\n\nA flow is identified by a pair of strings: the name of the FlowSchema\nand a \"flow distinguisher\" string.  The flow distinguisher is computed\nfrom the request according to a rule that is configured in the\nFlowSchema.\n\nEach FlowSchema has:\n- A boolean test of an authenticated request;\n- A matching precedence (default value is 1000);\n- A reference to a RequestPriority object; and\n- An optional rule for computing the request’s flow distinguisher; not\n  allowed for a FlowSchema that refers to a RequestPriority that is\n  exempt or has just one queue.\n\nEach RequestPriority has:\n- An `exempt` boolean (which defaults to `false`).\n- A `catchAll` boolean (which defaults to `false`), which is relevant\n  only to default behavior.\n\nEach non-exempt RequestPriority also has:\n- A non-negative integer AssuredConcurrencyShares;\n- A number of queues; and\n- A queue length limit.\n\nEach non-exempt RequestPriority with more than one queue also has:\n- A hand size (a small positive number).\n\nThe best matching FlowSchema for a given request is one of those whose\nboolean test accepts the request.  It is a configuration error if\nthere is no FlowSchema that matches every request.  In case multiple\nschemas accept the request, the best is one of those with the\nlogically highest matching precedence.  In case there are multiple of\nthose the implementation is free to pick any of those as best.  A\nmatching precedence is an integer, and a numerically lower number\nindicates a logically higher precedence.\n\nA FlowSchema’s boolean test is constructed from atomic tests.  Each\nfrom _either_ the client identity attributes or those that\n(scalar, pattern, or set).  For every available atomic test, its\ninverse is also available.  Atomic tests can be ANDed together.  Those\nconjunctions can then be ORed together.  The predicate of a FlowSchema\nis such a disjunction.\n\nA FlowSchema’s rule for computing the request’s flow distinguisher\nidentifies a string attribute of the authenticated request and\noptionally includes a transformation.  The available string attributes\nare (1) namespace of a resource-oriented request (available only if\nthe predicate accepts only resource-oriented requests) and (2)\nusername.  If no transformation is indicated then the flow\ndistinguisher is simply the selected request attribute.  There is only\none transformation available, and it is based on a regex that is\nconfigured in the flow schema and contains a capturing group.  The\ntransformation consists of doing a complete match against the regex\nand extracting submatch number 1; if the selected string does not\nmatch the regex then the transformation yields the empty string.\n\n### Assignment to a Queue\n\nA non-exempt RequestPriority object also has a number of queues (we\nare talking about a number here, not the actual set of queues; the\nqueues exist independently at each apiserver).  If the\nRequestPriority’s number of queues is more than one then the following\nlogic is used to assign a request to a queue.\n\nFor a given priority at a given apiserver, each queue is identified by\na numeric index (starting at zero).  A RequestPriority has a hand size\nH (so called because the technique here is an application of shuffle\nsharding), a small positive number.  When a request arrives at an\napiserver the request flow identifier’s string pair is hashed and the\nhash value is used to shuffle the queue indices and deal a hand of\nsize H, as follows.  We use a hash function that produces at least 64\nbits, and 64 of those bits are taken as an unsigned integer we will\ncall V.  The next step is finding the unique set of integers A[0] in\n[0, numQueues), A[1] in [0, numQueues-1), … A[H-1] in\n[0, numQueues-(H-1)), A[H] \u003e= 0 such that V = sum[i=0, 1, ...H] A[i] *\nff(numQueues, i), where ff(N, M) is the falling factorial N!/(N-M)!.\nThe probability distributions of each of these A’s will not be\nperfectly even, but we constrain the configuration such that\nff(numQueues, H) is less than 2^60 to keep the unevenness small.  Then\nthe coefficients A[0], … A[H-1] are converted into queue indices I[0],\n… I[H-1] as follows.  I[0] = A[0].  I[1] is the A[1]’th entry in the\nlist of queue indices excluding I[0].  I[2] is the A[2]’th entry in\nthe list of queue indices excluding I[0] and I[1].  And so on.\n\nThe lengths of the queues identified by I[0], I[1], … I[H-1] are\nexamined, and the request is put in one of the shortest queues.\n\nFor example, if a RequestPriority has numQueues=128 and handSize=6,\nthe hash value V is converted into 6 unique queue indices plus\n3905000064000*A[6].  There are 128 choose 6, which is about 5.4\nbillion, sets of 6 integers in the range [0,127].  Thus, if there is\none heavy flow and many light flows, the probability of a given light\nflow hashing to the same set of 6 queues as the heavy flow is about\none in 5.4 billion.\n\nIt is the queues that compete fairly.\n\nSince the assignment to queues is based on flows, a useful\nconfiguration will be one in which flows are meaningful boundaries for\nconfinement/competition.  For bad example, if a particular\nFlowSchema's flows are based on usernames and bad behavior correlates\nwith namespace then the bad behavior will be spread among all the\nqueues of that schema's priority.  Administrators need to make a good\nchoice for how flows are distinguished.\n\n\n#### Queue Assignment Proof of Concept\n\nThe following golang code shows a simple recursive technique to\nshuffle, deal, and pick.\n\n```go\npackage main\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"math/rand\"\n)\n\nvar numQueues uint64\n\nfunc shuffleDealAndPick(v, nq uint64,\n\tlengthOfQueue func(int) int,\n\tmr func( /*in [0, nq-1]*/ int) /*in [0, numQueues-1] and excluding previously determined members of I*/ int,\n\tnRem, minLen, bestIdx int) int {\n\tif nRem \u003c 1 {\n\t\treturn bestIdx\n\t}\n\tvNext := v / nq\n\tai := int(v - nq*vNext)\n\tii := mr(ai)\n\ti := numQueues - nq // i is used only for debug printing\n\tmrNext := func(a /*in [0, nq-2]*/ int) /*in [0, numQueues-1] and excluding I[0], I[1], ... ii*/ int {\n\t\tif a \u003c ai {\n\t\t\tfmt.Printf(\"mr[%v](%v) going low\\n\", i, a)\n\t\t\treturn mr(a)\n\t\t}\n\t\tfmt.Printf(\"mr[%v](%v) going high\\n\", i, a)\n\t\treturn mr(a + 1)\n\t}\n\tlenI := lengthOfQueue(ii)\n\tfmt.Printf(\"Considering A[%v]=%v, I[%v]=%v, qlen[%v]=%v\\n\\n\", i, ai, i, ii, i, lenI)\n\tif lenI \u003c minLen {\n\t\tminLen = lenI\n\t\tbestIdx = ii\n\t}\n\treturn shuffleDealAndPick(vNext, nq-1, lengthOfQueue, mrNext, nRem-1, minLen, bestIdx)\n}\n\nfunc main() {\n\tnumQueues = uint64(128)\n\thandSize := 6\n\thashValue := rand.Uint64()\n\tqueueIndex := shuffleDealAndPick(hashValue, numQueues, func (idx int) int {return idx % 10}, func(i int) int { return i }, handSize, math.MaxInt32, -1)\n\tfmt.Printf(\"For V=%v, numQueues=%v, handSize=%v, chosen queue is %v\\n\", hashValue, numQueues, handSize, queueIndex)\n}\n```\n\n#### Probability of Collisions\n\nThe following code tabulates some probabilities of collisions.\nSpecifically, if there are `nHands` elephants, `probNextCovered` is\nthe probability that a random mouse entirely collides with the\nelephants.  This is assuming fair dice and independent choices.  This\nis not exactly what we have, but is close.\n\n```go\npackage main\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n)\n\n// sum computes the sum of the given slice of numbers\nfunc sum(v []float64) float64 {\n\tc := append([]float64{}, v...)\n\tsort.Float64s(c) // to minimize loss of accuracy when summing\n\tvar s float64\n\tfor i := 0; i \u003c len(c); i++ {\n\t\ts += c[i]\n\t}\n\treturn s\n}\n\n// choose returns the number of subsets of size m of a set of size n\nfunc choose(n, m int) float64 {\n\tif m == 0 || m == n {\n\t\treturn 1\n\t}\n\tvar ans = float64(n)\n\tfor i := 1; i \u003c m; i++ {\n\t\tans = ans * float64(n-i) / float64(i+1)\n\t}\n\treturn ans\n}\n\n// nthDeal analyzes the result of another shuffle and deal in a series of shuffles and deals.\n// Each shuffle and deal randomly picks `handSize` distinct cards from a deck of size `deckSize`.\n// Each successive shuffle and deal is independent of previous deals.\n// `first` indicates that this is the first shuffle and deal.\n// `prevDist[nUnique]` is the probability that the number of unique cards previously dealt is `nUnique`,\n// and is unused when `first`.\n// `dist[nUnique]` is the probability that the number of unique cards dealt up through this deal is `nUnique`.\n// `distSum` is the sum of `dist`, and should be 1.\n// `expectedUniques` is the expected value of nUniques at the end of this deal.\n// `probNextCovered` is the probability that another shuffle and deal will deal only cards that have already been dealt.\nfunc nthDeal(first bool, handSize, deckSize int, prevDist []float64) (dist []float64, distSum, expectedUniques, probNextCovered float64) {\n\tdist = make([]float64, deckSize+1)\n\texpects := make([]float64, deckSize+1)\n\tnexts := make([]float64, deckSize+1)\n\tif first {\n\t\tdist[handSize] = 1\n\t\texpects[handSize] = float64(handSize)\n\t\tnexts[handSize] = 1 / choose(deckSize, handSize)\n\t} else {\n\t\tfor nUnique := handSize; nUnique \u003c= deckSize; nUnique++ {\n\t\t\tconts := make([]float64, handSize+1)\n\t\t\tfor news := 0; news \u003c= handSize; news++ {\n\t\t\t\t// one way to get to nUnique is for `news` new uniques to appear in this deal,\n\t\t\t\t// and all the previous deals to have dealt nUnique-news unique cards.\n\t\t\t\tprevUnique := nUnique - news\n\t\t\t\tways := choose(deckSize-prevUnique, news) * choose(prevUnique, handSize-news)\n\t\t\t\tconts[news] = ways * prevDist[prevUnique]\n\t\t\t\t//fmt.Printf(\"nUnique=%v, news=%v, ways=%v\\n\", nUnique, news, ways)\n\t\t\t}\n\t\t\tdist[nUnique] = sum(conts) / choose(deckSize, handSize)\n\t\t\texpects[nUnique] = dist[nUnique] * float64(nUnique)\n\t\t\tnexts[nUnique] = dist[nUnique] * choose(nUnique, handSize) / choose(deckSize, handSize)\n\t\t}\n\n\t}\n\treturn dist, sum(dist), sum(expects), sum(nexts)\n}\n\nfunc main() {\n\thandSize := 7\n\tdeckSize := 256\n\tfmt.Printf(\"choose(%v, %v) = %v\\n\", deckSize, handSize, choose(deckSize, handSize))\n\tvar dist []float64\n\tvar probNextCovered float64\n\tfor nHands := 1; probNextCovered \u003c 0.01; nHands++ {\n\t\tvar distSum, expected float64\n\t\tdist, distSum, expected, probNextCovered = nthDeal(nHands == 1, handSize, deckSize, dist)\n\t\tfmt.Printf(\"After %v hands, distSum=%v, expected=%v, probNextCovered=%v, dist=%v\\n\", nHands, distSum, expected, probNextCovered, dist)\n\t}\n}\n```\n\n\n### Resource Limits\n\n#### Primary CPU and Memory Protection\n\nThis proposal controls both CPU and memory consumption of running\nrequests by imposing a single concurrency limit per apiserver.  It is\nexpected that this concurrency limit can be set to a value that\nprovides effective protection of both CPU and memory while not being\ntoo low for either.\n\nThe configuration of an apiserver includes a concurrency limit.  This\nis a number, whose units is a number of readonly requests served\nconcurrently.  Unlike in today's max-in-flight handler, the mutating\nand readonly requests are commingled without distinction.  The primary\nresource limit applied is that at any moment in time the number of\nrunning non-exempt requests should not exceed the concurrency limit.\nRequests of an exempt priority are neither counted nor limited, as in\ntoday's max-in-flight handler.  For the remainder, each server's\noverall concurrency limit is divided among those non-exempt priority\nlevels and each enforces its own limit (independently of the other\nlevels).\n\nAt the first stage of development, an apiserver’s concurrency limit\nwill be derived from the existing configuration options for\nmax-mutating-in-flight and max-readonly-in-flight, by taking their\nsum.  Later we may migrate to a single direct configuration option.\nEven later we intend to automatomatically tune the setting of an\napiserver's concurrency limit.\n\n#### Secondary Memory Protection\n\nA RequestPriority is also configured with a limit on the number of\nrequests that may be waiting in a given queue.\n\n#### Latency Protection\n\nAn apiserver is also configured with a limit on the amount of time\nthat a request may wait in its queue.  If this time passes while a\nrequest is still waiting for service then the request will be\nrejected.\n\nprotection could keep us from violating latency SLOs even though we\nare dropping many requests.\n\n### Queuing\n\nOnce a request is categorized and assigned to a queue the next\ndecision is whether to reject or accept that request.\n\nA request of an exempt priority is never rejected and never waits in a\nqueue; such a request is dispatched as soon as it arrives.\n\nFor queuing requests of non-exempt priority, the first step is to\nreject all the requests that have been waiting longer than the\nconfigured limit.  Once that is done, the newly arrived request is\nconsidered.  This request is rejected if and only if the total number\nof requests waiting in its queue is at least the configured limit on\nthat number.\n\nA possible alternative would accept the request unconditionally and,\nif that made the queue too long, reject the request at the head of the\nqueue.  That would be the preferred design if we were confident that\nrejection will cause the client to slow down.  Lacking that\nconfidence, we choose to reject the youngest rather than the oldest\nrequest of the queue, so that an investment in holding a request in a\nqueue has a chance of eventually getting useful work done.\n\n### Dispatching\n\nRequests of an exempt priority are never held up in a queue; they are\nalways dispatched immediately.  Following is how the other requests\nare dispatched at a given apiserver.\n\nThe concurrency limit of an apiserver is divided among the non-exempt\npriority levels in proportion to their assured concurrency shares.\nThis produces the assured concurrency value (ACV) for each non-exempt\npriority level:\n\n```\nACV(l) = ceil( SCL * ACS(l) / ( sum[priority levels k] ACS(k) ) )\n```\n\nwhere SCL is the apiserver's concurrency limit and ACS(l) is the\nAssuredConcurrencyShares for priority level l.\n\nDispatching is done independently for each priority level.  Whenever\n(1) a non-exempt priority level's number of running requests is below\nthe level's assured concurrency value and (2) that priority level has\na non-empty queue, it is time to dispatch another request for service.\nThe Fair Queuing for Server Requests algorithm below is used to pick a\nnon-empty queue at that priority level.  Then the request at the head\nof that queue is dispatched.\n\n\n#### Fair Queuing for Server Requests\n\nThis is based on fair queuing but is modified to deal with serving\nrequests in an apiserver instead of transmitting packets in a router.\nYou can find the original fair queuing paper at\n[ACM](https://dl.acm.org/citation.cfm?doid=75247.75248) or\n[MIT](http://people.csail.mit.edu/imcgraw/links/research/pubs/networks/WFQ.pdf),\nand an\n[implementation outline at Wikipedia](https://en.wikipedia.org/wiki/Fair_queuing).\nOur problem differs from the normal fair queuing problem in three\nways.  One is that we are dispatching requests to be served rather\nthan packets to be transmitted.  Another difference is that multiple\nrequests may be served at once.  The third difference is that the\nactual service time (i.e., duration) is not known until a request is\ndone being served.  The first two differences can easily be handled by\nstraightforward adaptation of the concept called \"R(t)\" in the\noriginal paper and \"virtual time\" in the implementation outline.  In\nthat implementation outline, the notation `now()` is used to mean\nreading the _virtual_ clock.  In the original paper’s terms, \"R(t)\" is\nthe number of \"rounds\" that have been completed at real time t, where\na round consists of virtually transmitting one bit from every\nnon-empty queue in the router (regardless of which queue holds the\npacket that is really being transmitted at the moment); in this\nconception, a packet is considered to be \"in\" its queue until the\npacket’s transmission is finished.  For our problem, we can define a\nround to be giving one nanosecond of CPU to every non-empty queue in\nthe apiserver (where emptiness is judged based on both queued and\nexecuting requests from that queue), and define R(t) = (server start\ntime) + (1 ns) * (number of rounds since server start).  Let us write\nNEQ(t) for that number of non-empty queues in the apiserver at time t.\nFor a given queue \"q\", let us also write \"reqs(q, t)\" for the number\nof requests of that queue at that time.  Let us also write C for the\nconcurrency limit.  At a particular time t, the partial derivative of\nR(t) with respect to t is\n\n```\nmin(sum[over q] reqs(q, t), C) / NEQ(t) .\n```\n\nIn terms of the implementation outline, this is the rate at which\nvirtual time (`now()`) is advancing at time t (in virtual nanoseconds\nper real nanosecond).  Where the implementation outline adds packet\nsize to a virtual time, in our version this corresponds to adding a\nservice time (i.e., duration) to virtual time.\n\nThe third difference is handled by modifying the algorithm to dispatch\nbased on an initial guess at the request’s service time (duration) and\nthen make the corresponding adjustments once the request’s actual\nservice time is known.  This is similar, although not exactly\nisomorphic, to the original paper’s adjustment by δ for the sake of\npromptness.\n\nFor implementation simplicity (see below), let us use the same initial\nservice time guess for every request; call that duration G.  A good\nchoice might be the service time limit (1 minute).  Different guesses\nwill give slightly different dynamics, but any positive number can be\nused for G without ruining the long-term behavior.\n\nAs in ordinary fair queuing, there is a bound on divergence from the\nideal.  In plain fair queuing the bound is one packet; in our version\nit is C requests.\n\nTo support efficiently making the necessary adjustments once a\nrequest’s actual service time is known, the virtual finish time of a\nrequest and the last virtual finish time of a queue are not\nrepresented directly but instead computed from queue length, request\nposition in the queue, and an alternate state variable that holds the\nqueue’s virtual start time.  While the queue is empty and has no\nrequests executing: the value of its virtual start time variable is\nignored and its last virtual finish time is considered to be in the\nvirtual past.  When a request arrives to an empty queue with no\nrequests executing, the queue’s virtual start time is set to `now()`.\nThe virtual finish time of request number J in the queue (counting\nfrom J=1 for the head) is J * G + (virtual start time).  While the\nqueue is non-empty: the last virtual finish time of the queue is the\nvirtual finish time of the last request in the queue.  While the queue\nis empty and has a request executing: the last virtual finish time is\nthe queue’s virtual start time.  When a request is dequeued for\nservice the queue’s virtual start time is advanced by G.  When a\nrequest finishes being served, and the actual service time was S, the\nqueue’s virtual start time is decremented by G - S.\n\n### Example Configuration\n\n\nFor requests from admins and requests in service of other, potentially\nsystem, requests.\n```yaml\nkind: RequestPriority\nmeta:\n  name: system-top\nspec:\n  exempt: true\n```\n\nFor system self-maintenance requests.\n```yaml\nkind: RequestPriority\nmeta:\n  name: system-high\nspec:\n  assuredConcurrencyShares: 100\n  queues: 128\n  handSize: 6\n  queueLengthLimit: 100\n```\n\nFor the garbage collector.\n```yaml\nkind: RequestPriority\nmeta:\n  name: system-low\nspec:\n  assuredConcurrencyShares: 30\n  queues: 1\n  queueLengthLimit: 1000\n```\n\nFor user requests from kubectl.\n```yaml\nkind: RequestPriority\nmeta:\n  name: workload-high\nspec:\n  assuredConcurrencyShares: 30\n  queues: 128\n  handSize: 6\n  queueLengthLimit: 100\n```\n\nFor requests from controllers processing workload.\n```yaml\nkind: RequestPriority\nmeta:\n  name: workload-low\nspec:\n  catchAll: true\n  assuredConcurrencyShares: 100\n  queues: 128\n  handSize: 6\n  queueLengthLimit: 100\n```\n\nSome flow schemata.\n\n```yaml\nkind: FlowSchema\nmeta:\n  name: system-top\nspec:\n  requestPriority:\n    name: system-top\n  match:\n  - and: # writes by admins (does this cover loopback too?)\n    - superSet:\n      field: groups\n      set: [ \"system:masters\" ]\n```\n\n```yaml\nkind: FlowSchema\nmeta:\n  name: system-high\nspec:\n  requestPriority:\n    name: system-high\n  flowDistinguisher:\n    source: user\n    # no transformation in this case\n  match:\n  - and: # heartbeats by nodes\n    - superSet:\n      field: groups\n      set: [ \"system:nodes\" ]\n    - equals:\n      field: resource\n      value: nodes\n  - and: # kubelet and kube-proxy ops on system objects\n    - superSet:\n      field: groups\n      set: [ \"system:nodes\" ]\n    - equals:\n      field: namespace\n      value: kube-system\n  - and: # leader elections for system controllers\n    - patternMatch:\n      field: user\n      pattern: system:controller:.*\n    - inSet:\n      field: resource\n      set: [ \"endpoints\", \"configmaps\", \"leases\" ]\n    - equals:\n      field: namespace\n      value: kube-system\n```\n\n```yaml\nkind: FlowSchema\nmeta:\n  name: system-low\nspec:\n  matchingPriority: 900\n  requestPriority:\n    name: system-low\n  flowDistinguisher:\n    source: user\n    # no transformation in this case\n  match:\n  - and: # the garbage collector\n    - equals:\n      field: user\n      value: system:controller:garbage-collector\n```\n\n```yaml\nkind: FlowSchema\nmeta:\n  name: workload-high\nspec:\n  requestPriority:\n    name: workload-high\n  flowDistinguisher:\n    source: namespace\n    # no transformation in this case\n  match:\n  - and: # users using kubectl\n    - notPatternMatch:\n      field: user\n      pattern: system:serviceaccount:.*\n```\n\n```yaml\nkind: FlowSchema\nmeta:\n  name: workload-low\nspec:\n  matchingPriority: 9999\n  requestPriority:\n    name: workload-high\n  flowDistinguisher:\n    source: namespace\n    # no transformation in this case\n  match:\n  - and: [ ] # match everything\n```\n  \nFollowing is a FlowSchema that might be used for the requests by the\naggregated apiservers of\nhttps://github.com/MikeSpreitzer/kube-examples/tree/add-kos/staging/kos\nto create TokenReview and SubjectAccessReview objects.\n\n\n```\nkind: FlowSchema\nmeta:\n  name: system-top\nspec:\n  matchingPriority: 900\n  requestPriority:\n    name: system-top\n  flowDistinguisher:\n    source: user\n    # no transformation in this case\n  match:\n  - and:\n    - inSet:\n      field: resource\n      set: [ \"tokenreviews\", \"subjectaccessreviews\" ]\n    - superSet:\n      field: user\n      set: [ \"system:serviceaccount:example-com:network-apiserver\" ]\n```\n\n### Reaction to Configuration Changes\n\nWe do not seek to make our life easy by making any configuration\nobject fields immutable.  Recall that objects can be deleted and\nreplacements (with the same name) created, so server-enforced\nimmutability of field values provides no useful guarantee to\ndownstream consumers (such as the request filter here).  We could try\nto get useful guarantees by adding a finalizer per (config object,\napiserver), but at this level of development we will not attempt that.\n\nMany details of the configuration are consumed only momentarily;\nchanges in these pose no difficulty.  Challenging changes include\nchanges in the number of queues, the queue length limit, or the\nassured concurrency value (which is derived from several pieces of\ndeletion of a priority level itself.\n\nAn increase in the number of queues of a priority level is handled by\nsimply adding queues.  A decrease is handled by making a distinction\nbetween the desired and the actual number of queues.  When the desired\nnumber drops below the actual number, the undesired queues are left in\nplace until they are naturally drained; new requests are put in only\nthe desired queues.  When an undesired queue becomes empty it is\ndeleted and the fair queuing round-robin pointer is advanced if it was\npointing to that queue.\n\nWhen the assured concurrency value of a priority level increases,\nadditional requests are dispatched if possible.  When the assured\nfilter does not abort or suspend requests that are currently being\nserved.\n\nWhen the queue length limit of a priority level increases, no\nimmediate reaction is required.  When the queue length limit\nlonger than the new length limit are left to naturally shrink as they\nare drained by dispatching and timeouts.\n\nWhen a request priority configuration object is deleted, in a given\napiserver the corresponding implementation objects linger until all\nthe queues of that priority level are empty.  A FlowSchema associated\nwith one of these lingering undesired priority levels matches no\nrequests.\n\nThe [Dispatching](#Dispatching) section prescribes how the assured\nconcurrency value (`ACV`) is computed for each priority level, and the\nsum there is over all the _desired_ priority levels (i.e., excluding\nthe lingering undesired priority levels).  For this reason and for\nothers, at any given time this may compute for some priority level(s)\nan assured concurrency value that is lower than the number currently\nexecuting.  In these situations the total number allowed to execute\nwill temporarily exceed the apiserver's configured concurrency limit\n(`SCL`) and will settle down to the configured limit as requests\ncomplete their service.\n\n### Default Behavior\n\nThere must be reasonable behavior \"out of the box\", and it should be\nat least a little difficult for an administrator to lock himself out\nof this subsystem.  To accomplish these things there are two levels of\ndefaulting: one concerning behavior, and one concerning explicit API\nobjects.\n\nThe effective configuration is the union of (a) the actual API objects\nthat exist and (b) implicitly generated backstop objects.  The latter\nare not actual API objects, and might not ever exist as identifiable\nobjects in the implementation, but are figments of our imagination\nused to describe the behavior of this subsystem.  These backstop\nobjects are implicitly present and affecting behavior when needed.\nThere are two implicitly generated RequestPriority backstop objects.\nOne is equivalent to the `system-top` object exhibited above, and it\nexists while there is no actual RequestPriority object with `exempt ==\ntrue`.  The other is equivalent to the `workload-low` object exhibited\nabove, and exists while there is no RequestPriority object with\nnon-exempt priority.  There are also two implicitly generated\nFlowSchema backup objects.  Whenever a request whose groups include\n`system:masters` is not matched by any actual FlowSchema object, a\nbackstop equivalent to the `system-top` object exhibited above is\nconsidered to exist.  Whenever a request whose groups do not include\n`system:masters` is not matched by any actual FlowSchema object, the\nfollowing backstop object is considered to exist.\n\n```yaml\nkind: FlowSchema\nmeta:\n  name: non-top-backstop\nspec:\n  matchingPriority: (doesn’t really matter)\n  requestPriority:\n    name: (name of an effectively existing RequestPriority, whether\n           that is real or backstop, with catchAll==true)\n  flowDistinguisher:\n    source: user\n    # no transformation in this case\n  match:\n  - and: [ ] # match everything\n```\n\nThe other part of the defaulting story concerns making actual API\nobjects exist, and it goes as follows.  Whenever there is no actual\nRequestPriority object with `exempt == true`, the RequestPriority\nalready in use by an existing RequestPriority object.  Whenever there\nis no actual FlowSchema object that refers to an exempt\nRequestPriority object, the schema objects shown above as examples are\n\n### Prometheus Metrics\n\nPrior to this KEP, the relevant available metrics from an apiserver are:\n- apiserver_current_inflight_requests (gauge, broken down by mutating or not)\n- apiserver_longrunning_gauge\n- apiserver_request_count (cumulative number served)\n- apiserver_request_latencies (histogram)\n- apiserver_request_latencies_summary\n\nThis KEP adds the following metrics.\n- apiserver_rejected_requests (count, broken down by priority, FlowSchema, when (arrival vs timeout))\n- apiserver_current_inqueue_requests (gauge, broken down by priority, FlowSchema)\n- apiserver_request_queue_length (histogram, broken down by\n  RequestPriority name; buckets set at 0, 0.25, 0.5, 0.75, 0.9, 1.0\n  times the relevant queue length limit)\n- apiserver_current_executing_requests (gauge, broken down by priority, FlowSchema)\n- apiserver_dispatched_requests (count, broken down by priority, FlowSchema)\n- apiserver_wait_duration (histogram, broken down by priority, FlowSchema)\n- apiserver_service_duration (histogram, broken down by priority, FlowSchema)\n\n### Testing\n\nThere should be one or more end-to-end tests that exercise the\nfunctionality introduced by this KEP.  Following are a couple of\nsuggestions.\n\nOne simple test would be to use a client like\nhttps://github.com/MikeSpreitzer/k8api-scaletest/tree/master/cmdriverclosed\nto drive workload with more concurrency than is configured to be\nadmitted, and see whether the amount admitted is as configured.\n\nA similar but more sophisticated test would be like the ConfigMap\ndriver but would create/update/delete objects that have some\nnon-trivial behavior associated with them.  One possibility would be\nServiceAccount objects.  Creation of a ServiceAccount object implies\ncreation of a Secret, and deletion also has an implication.  Thrashing\nsuch objects would test that the workload does not crowd out the\ngarbage collector.\n\n\n### Observed Requests\n\nTo provide data about requests to use in designing the configuration,\nhere are some observations from a running system late in the\nrelease 1.16 development cycle.  These are extracts from the\nkube-apiserver log file, with linebreaks and indentation added for\nreadability.\n\nThe displayed data are a `RequestInfo` from\n`k8s.io/apiserver/pkg/endpoints/request` and an `Info` from\n`k8s.io/apiserver/pkg/authentication/user`.\n\n#### Loopback\n\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/apis/admissionregistration.k8s.io/v1beta1/mutatingwebhookconfigurations\",\n  Verb:\"list\", APIPrefix:\"apis\",\n  APIGroup:\"admissionregistration.k8s.io\", APIVersion:\"v1beta1\",\n  Namespace:\"\", Resource:\"mutatingwebhookconfigurations\",\n  Subresource:\"\", Name:\"\",\n  Parts:[]string{\"mutatingwebhookconfigurations\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:apiserver\",\n  UID:\"388b748d-481c-4348-9c94-b7aab0c6efad\",\n  Groups:[]string{\"system:masters\"}, Extra:map[string][]string(nil)}\n```\n\n```\nRequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/api/v1/services\", Verb:\"watch\", APIPrefix:\"api\", APIGroup:\"\",\n  APIVersion:\"v1\", Namespace:\"\", Resource:\"services\", Subresource:\"\",\n  Name:\"\", Parts:[]string{\"services\"}},\nuser.Info=\u0026user.DefaultInfo{Name:\"system:apiserver\",\n  UID:\"388b748d-481c-4348-9c94-b7aab0c6efad\",\n  Groups:[]string{\"system:masters\"}, Extra:map[string][]string(nil)}\n```\n\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/api/v1/namespaces/default/services/kubernetes\", Verb:\"get\",\n  APIPrefix:\"api\", APIGroup:\"\", APIVersion:\"v1\", Namespace:\"default\",\n  Resource:\"services\", Subresource:\"\", Name:\"kubernetes\",\n  Parts:[]string{\"services\", \"kubernetes\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:apiserver\",\n  UID:\"388b748d-481c-4348-9c94-b7aab0c6efad\",\n  Groups:[]string{\"system:masters\"}, Extra:map[string][]string(nil)}\n  ```\n\n#### TokenReview from kube-controller-manager\n\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/apis/authentication.k8s.io/v1/tokenreviews\", Verb:\"create\",\n  APIPrefix:\"apis\", APIGroup:\"authentication.k8s.io\", APIVersion:\"v1\",\n  Namespace:\"\", Resource:\"tokenreviews\", Subresource:\"\", Name:\"\",\n  Parts:[]string{\"tokenreviews\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:kube-controller-manager\",\n  UID:\"\", Groups:[]string{\"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\n#### SubjectAccessReview by Aggregated API Server\n\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/apis/authorization.k8s.io/v1beta1/subjectaccessreviews\",\n  Verb:\"create\", APIPrefix:\"apis\", APIGroup:\"authorization.k8s.io\",\n  APIVersion:\"v1beta1\", Namespace:\"\", Resource:\"subjectaccessreviews\",\n  Subresource:\"\", Name:\"\", Parts:[]string{\"subjectaccessreviews\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:serviceaccount:example-com:network-apiserver\",\n  UID:\"55aa7599-67e7-4f29-80ae-d8c2cb5fd0c4\",\n  Groups:[]string{\"system:serviceaccounts\",\n    \"system:serviceaccounts:example-com\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\n#### GET of Custom Resource by Administrator Using Kubectl\n\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:false,\n  Path:\"/openapi/v2\", Verb:\"get\", APIPrefix:\"\", APIGroup:\"\",\n  APIVersion:\"\", Namespace:\"\", Resource:\"\", Subresource:\"\", Name:\"\",\n  Parts:[]string(nil)},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:admin\", UID:\"\",\n  Groups:[]string{\"system:masters\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/apis/network.example.com/v1alpha1/namespaces/default/networkattachments\",\n  Verb:\"list\", APIPrefix:\"apis\", APIGroup:\"network.example.com\",\n  APIVersion:\"v1alpha1\", Namespace:\"default\",\n  Resource:\"networkattachments\", Subresource:\"\", Name:\"\",\n  Parts:[]string{\"networkattachments\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:admin\", UID:\"\",\n  Groups:[]string{\"system:masters\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\n#### Node to Self\n\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/api/v1/nodes/127.0.0.1/status\", Verb:\"patch\", APIPrefix:\"api\",\n  APIGroup:\"\", APIVersion:\"v1\", Namespace:\"\", Resource:\"nodes\",\n  Subresource:\"status\", Name:\"127.0.0.1\", Parts:[]string{\"nodes\",\n    \"127.0.0.1\", \"status\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:node:127.0.0.1\", UID:\"\",\n  Groups:[]string{\"system:nodes\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/127.0.0.1\",\n  Verb:\"update\", APIPrefix:\"apis\", APIGroup:\"coordination.k8s.io\",\n  APIVersion:\"v1\", Namespace:\"kube-node-lease\", Resource:\"leases\",\n  Subresource:\"\", Name:\"127.0.0.1\", Parts:[]string{\"leases\",\n    \"127.0.0.1\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:node:127.0.0.1\", UID:\"\",\n  Groups:[]string{\"system:nodes\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\n#### Other Leases\n\nLIST by kube-controller-manager\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/apis/coordination.k8s.io/v1/leases\", Verb:\"list\",\n  APIPrefix:\"apis\", APIGroup:\"coordination.k8s.io\", APIVersion:\"v1\",\n  Namespace:\"\", Resource:\"leases\", Subresource:\"\", Name:\"\",\n  Parts:[]string{\"leases\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:kube-controller-manager\",\n  UID:\"\", Groups:[]string{\"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\nWATCH by kube-controller-manager\n```\nRequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/apis/coordination.k8s.io/v1beta1/leases\", Verb:\"watch\",\n  APIPrefix:\"apis\", APIGroup:\"coordination.k8s.io\",\n  APIVersion:\"v1beta1\", Namespace:\"\", Resource:\"leases\", Subresource:\"\",\n  Name:\"\", Parts:[]string{\"leases\"}},\nuser.Info=\u0026user.DefaultInfo{Name:\"system:kube-controller-manager\",\n  UID:\"\", Groups:[]string{\"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\n#### Status Update From System Controller To System Object\n\nDeployment controller\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/apis/apps/v1/namespaces/kube-system/deployments/kube-dns/status\",\n  Verb:\"update\", APIPrefix:\"apis\", APIGroup:\"apps\", APIVersion:\"v1\",\n  Namespace:\"kube-system\", Resource:\"deployments\", Subresource:\"status\",\n  Name:\"kube-dns\", Parts:[]string{\"deployments\", \"kube-dns\", \"status\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:serviceaccount:kube-system:deployment-controller\",\n  UID:\"2b126368-be77-454d-8893-0384f9895f02\",\n  Groups:[]string{\"system:serviceaccounts\",\n    \"system:serviceaccounts:kube-system\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\n#### Etcd Operator\n\nList relevant pods\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/api/v1/namespaces/example-com/pods\", Verb:\"list\",\n  APIPrefix:\"api\", APIGroup:\"\", APIVersion:\"v1\",\n  Namespace:\"example-com\", Resource:\"pods\", Subresource:\"\", Name:\"\",\n  Parts:[]string{\"pods\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:serviceaccount:example-com:default\",\n  UID:\"6c906aa7-135c-4094-8611-fbdb1a8ea077\",\n  Groups:[]string{\"system:serviceaccounts\",\n    \"system:serviceaccounts:example-com\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\nUpdate an etcd cluster\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/apis/etcd.database.coreos.com/v1beta2/namespaces/example-com/etcdclusters/the-etcd-cluster\",\n  Verb:\"update\", APIPrefix:\"apis\", APIGroup:\"etcd.database.coreos.com\",\n  APIVersion:\"v1beta2\", Namespace:\"example-com\",\n  Resource:\"etcdclusters\", Subresource:\"\", Name:\"the-etcd-cluster\",\n  Parts:[]string{\"etcdclusters\", \"the-etcd-cluster\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:serviceaccount:example-com:default\",\n  UID:\"6c906aa7-135c-4094-8611-fbdb1a8ea077\",\n  Groups:[]string{\"system:serviceaccounts\",\n    \"system:serviceaccounts:example-com\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\nKube-scheduler places an etcd Pod\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/api/v1/namespaces/example-com/pods/the-etcd-cluster-mxcxvgbcfg/binding\",\n  Verb:\"create\", APIPrefix:\"api\", APIGroup:\"\", APIVersion:\"v1\",\n  Namespace:\"example-com\", Resource:\"pods\", Subresource:\"binding\",\n  Name:\"the-etcd-cluster-mxcxvgbcfg\", Parts:[]string{\"pods\",\n    \"the-etcd-cluster-mxcxvgbcfg\", \"binding\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:kube-scheduler\", UID:\"\",\n  Groups:[]string{\"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\n#### Garbage Collectors\n\nPod GC, list nodes.\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/api/v1/nodes\", Verb:\"list\", APIPrefix:\"api\", APIGroup:\"\",\n  APIVersion:\"v1\", Namespace:\"\", Resource:\"nodes\", Subresource:\"\",\n  Name:\"\", Parts:[]string{\"nodes\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:serviceaccount:kube-system:pod-garbage-collector\",\n  UID:\"85b105c6-ca2f-42f0-8a75-1da13a8c1f6d\",\n  Groups:[]string{\"system:serviceaccounts\",\n    \"system:serviceaccounts:kube-system\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\nGeneric GC, `GET /api`\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:false, Path:\"/api\",\n  Verb:\"get\", APIPrefix:\"\", APIGroup:\"\", APIVersion:\"\", Namespace:\"\",\n  Resource:\"\", Subresource:\"\", Name:\"\", Parts:[]string(nil)},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:serviceaccount:kube-system:generic-garbage-collector\",\n  UID:\"61271d97-a959-467a-afd5-892dfd30dec9\",\n  Groups:[]string{\"system:serviceaccounts\",\n    \"system:serviceaccounts:kube-system\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\nGeneric GC, `GET /apis/coordination.k8s.io/v1beta1`\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:false,\n  Path:\"/apis/coordination.k8s.io/v1beta1\", Verb:\"get\",\n  APIPrefix:\"apis\", APIGroup:\"\", APIVersion:\"\", Namespace:\"\",\n  Resource:\"\", Subresource:\"\", Name:\"\", Parts:[]string(nil)},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:serviceaccount:kube-system:generic-garbage-collector\",\n  UID:\"61271d97-a959-467a-afd5-892dfd30dec9\",\n  Groups:[]string{\"system:serviceaccounts\",\n    \"system:serviceaccounts:kube-system\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\n#### Kube-Scheduler\n\nLIST\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/apis/storage.k8s.io/v1/storageclasses\", Verb:\"list\",\n  APIPrefix:\"apis\", APIGroup:\"storage.k8s.io\", APIVersion:\"v1\",\n  Namespace:\"\", Resource:\"storageclasses\", Subresource:\"\", Name:\"\",\n  Parts:[]string{\"storageclasses\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:kube-scheduler\", UID:\"\",\n  Groups:[]string{\"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\nWATCH\n```\nRequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/api/v1/services\", Verb:\"watch\", APIPrefix:\"api\", APIGroup:\"\",\n  APIVersion:\"v1\", Namespace:\"\", Resource:\"services\", Subresource:\"\",\n  Name:\"\", Parts:[]string{\"services\"}},\nuser.Info=\u0026user.DefaultInfo{Name:\"system:kube-scheduler\", UID:\"\",\n  Groups:[]string{\"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\nBind Pod of Hosted Control Plane\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/api/v1/namespaces/example-com/pods/the-etcd-cluster-mxcxvgbcfg/binding\",\n  Verb:\"create\", APIPrefix:\"api\", APIGroup:\"\", APIVersion:\"v1\",\n  Namespace:\"example-com\", Resource:\"pods\", Subresource:\"binding\",\n  Name:\"the-etcd-cluster-mxcxvgbcfg\", Parts:[]string{\"pods\",\n    \"the-etcd-cluster-mxcxvgbcfg\", \"binding\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:kube-scheduler\", UID:\"\",\n  Groups:[]string{\"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\nUpdate Status of System Pod\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/api/v1/namespaces/kube-system/pods/kube-dns-5f7bc9fd5c-2bsz8/status\",\n  Verb:\"update\", APIPrefix:\"api\", APIGroup:\"\", APIVersion:\"v1\",\n  Namespace:\"kube-system\", Resource:\"pods\", Subresource:\"status\",\n  Name:\"kube-dns-5f7bc9fd5c-2bsz8\", Parts:[]string{\"pods\",\n    \"kube-dns-5f7bc9fd5c-2bsz8\", \"status\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:kube-scheduler\", UID:\"\",\n  Groups:[]string{\"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\n Create Event About Hosted Control Plane\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/apis/events.k8s.io/v1beta1/namespaces/example-com/events\",\n  Verb:\"create\", APIPrefix:\"apis\", APIGroup:\"events.k8s.io\",\n  APIVersion:\"v1beta1\", Namespace:\"example-com\", Resource:\"events\",\n  Subresource:\"\", Name:\"\", Parts:[]string{\"events\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:kube-scheduler\", UID:\"\",\n  Groups:[]string{\"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\n#### Kubelet Updates Pod Status\n\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/api/v1/namespaces/default/pods/bb1-66bdc74b9c-bgm47/status\",\n  Verb:\"patch\", APIPrefix:\"api\", APIGroup:\"\", APIVersion:\"v1\",\n  Namespace:\"default\", Resource:\"pods\", Subresource:\"status\",\n  Name:\"bb1-66bdc74b9c-bgm47\", Parts:[]string{\"pods\",\n    \"bb1-66bdc74b9c-bgm47\", \"status\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:node:127.0.0.1\", UID:\"\",\n  Groups:[]string{\"system:nodes\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\n#### Controller in Hosted Control Plane\n\nLIST\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/apis/network.example.com/v1alpha1/subnets\", Verb:\"list\",\n  APIPrefix:\"apis\", APIGroup:\"network.example.com\",\n  APIVersion:\"v1alpha1\", Namespace:\"\", Resource:\"subnets\",\n  Subresource:\"\", Name:\"\", Parts:[]string{\"subnets\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:serviceaccount:example-com:kos-controller-manager\",\n  UID:\"cd95ccf8-f1ea-4398-9cc3-035331125442\",\n  Groups:[]string{\"system:serviceaccounts\",\n   \"system:serviceaccounts:example-com\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\nWATCH\n```\nRequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/apis/network.example.com/v1alpha1/subnets\", Verb:\"watch\",\n  APIPrefix:\"apis\", APIGroup:\"network.example.com\",\n  APIVersion:\"v1alpha1\", Namespace:\"\", Resource:\"subnets\",\n  Subresource:\"\", Name:\"\", Parts:[]string{\"subnets\"}},\nuser.Info=\u0026user.DefaultInfo{Name:\"system:serviceaccount:example-com:kos-controller-manager\",\n  UID:\"cd95ccf8-f1ea-4398-9cc3-035331125442\",\n  Groups:[]string{\"system:serviceaccounts\",\n    \"system:serviceaccounts:example-com\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\nUPDATE\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/apis/network.example.com/v1alpha1/namespaces/default/subnets/sn-1\",\n  Verb:\"update\", APIPrefix:\"apis\", APIGroup:\"network.example.com\",\n  APIVersion:\"v1alpha1\", Namespace:\"default\", Resource:\"subnets\",\n  Subresource:\"\", Name:\"sn-1\", Parts:[]string{\"subnets\", \"sn-1\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:serviceaccount:example-com:kos-controller-manager\",\n  UID:\"cd95ccf8-f1ea-4398-9cc3-035331125442\",\n  Groups:[]string{\"system:serviceaccounts\",\n    \"system:serviceaccounts:example-com\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\n#### LOG and EXEC on Workload Pod\n\n`kubectl logs bb1-66bdc74b9c-bgm47`\n\n```\nRequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/api/v1/namespaces/default/pods/bb1-66bdc74b9c-bgm47/log\",\n  Verb:\"get\", APIPrefix:\"api\", APIGroup:\"\", APIVersion:\"v1\",\n  Namespace:\"default\", Resource:\"pods\", Subresource:\"log\",\n  Name:\"bb1-66bdc74b9c-bgm47\", Parts:[]string{\"pods\",\n    \"bb1-66bdc74b9c-bgm47\", \"log\"}},\nuser.Info=\u0026user.DefaultInfo{Name:\"system:admin\", UID:\"\",\n  Groups:[]string{\"system:masters\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\n`kubectl logs -f bb1-66bdc74b9c-bgm47`\n\n```\nRequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/api/v1/namespaces/default/pods/bb1-66bdc74b9c-bgm47/log\",\n  Verb:\"get\", APIPrefix:\"api\", APIGroup:\"\", APIVersion:\"v1\",\n  Namespace:\"default\", Resource:\"pods\", Subresource:\"log\",\n  Name:\"bb1-66bdc74b9c-bgm47\", Parts:[]string{\"pods\",\n    \"bb1-66bdc74b9c-bgm47\", \"log\"}},\nuser.Info=\u0026user.DefaultInfo{Name:\"system:admin\", UID:\"\",\n  Groups:[]string{\"system:masters\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\n`kubectl exec bb1-66bdc74b9c-bgm47 date`\n\n```\nRequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/api/v1/namespaces/default/pods/bb1-66bdc74b9c-bgm47/exec\",\n  Verb:\"create\", APIPrefix:\"api\", APIGroup:\"\", APIVersion:\"v1\",\n  Namespace:\"default\", Resource:\"pods\", Subresource:\"exec\",\n  Name:\"bb1-66bdc74b9c-bgm47\", Parts:[]string{\"pods\",\n    \"bb1-66bdc74b9c-bgm47\", \"exec\"}},\nuser.Info=\u0026user.DefaultInfo{Name:\"system:admin\", UID:\"\",\n  Groups:[]string{\"system:masters\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\n#### Requests Over Insecure Port\n\nBefore fix\n```\nW0821 18:10:40.546166    5468 reqmgmt.go:56] No User found in context\n\u0026context.valueCtx{Context:(*context.valueCtx)(0xc007c89ad0), key:0,\n  val:(*request.RequestInfo)(0xc0016afa20)}\nof request \u0026http.Request{Method:\"GET\",\n  URL:(*url.URL)(0xc006f87d00), Proto:\"HTTP/1.1\", ProtoMajor:1,\n  ProtoMinor:1,\n  Header:http.Header{\"Accept\":[]string{\"*/*\"},\n    \"User-Agent\":[]string{\"curl/7.58.0\"}},\n  Body:http.noBody{}, GetBody:(func() (io.ReadCloser, error))(nil),\n  ContentLength:0, TransferEncoding:[]string(nil), Close:false,\n  Host:\"localhost:8080\", Form:url.Values(nil),\n  PostForm:url.Values(nil),\n  MultipartForm:(*multipart.Form)(nil), Trailer:http.Header(nil),\n  RemoteAddr:\"127.0.0.1:41628\", RequestURI:\"/healthz\",\n  TLS:(*tls.ConnectionState)(nil), Cancel:(\u003c-chan struct {})(nil),\n  Response:(*http.Response)(nil), ctx:(*context.valueCtx)(0xc007c89b00)}\n```\n\nAfter https://github.com/kubernetes/kubernetes/pull/81788\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:false,\n  Path:\"/healthz/zzz\", Verb:\"get\", APIPrefix:\"\", APIGroup:\"\",\n  APIVersion:\"\", Namespace:\"\", Resource:\"\", Subresource:\"\", Name:\"\",\n  Parts:[]string(nil)},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:unsecured\", UID:\"\",\n  Groups:[]string{\"system:masters\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\n```\nrequestInfo=\u0026request.RequestInfo{IsResourceRequest:true,\n  Path:\"/api/v1/namespaces/fooobar\", Verb:\"get\", APIPrefix:\"api\",\n  APIGroup:\"\", APIVersion:\"v1\", Namespace:\"fooobar\",\n  Resource:\"namespaces\", Subresource:\"\", Name:\"fooobar\",\n  Parts:[]string{\"namespaces\", \"fooobar\"}},\nuserInfo=\u0026user.DefaultInfo{Name:\"system:unsecured\", UID:\"\",\n  Groups:[]string{\"system:masters\", \"system:authenticated\"},\n  Extra:map[string][]string(nil)}\n```\n\n### Implementation Details/Notes/Constraints\n\nTBD\n\n### Risks and Mitigations\n\nImplementing this KEP will increase the overhead in serving each\nrequest, perhaps to a degree that depends on some measure of system\nand/or workload size.  The additional overhead must be practically\nlimited.\n\nThere are likely others.\n\n## Design Details\n\nWe are still ironing out the high level goals and approach.  Several\nearlier proposals have been floated, as listed next.  This section\ncontains a discussion of the issues.\n\n### References\n\n- [Min Kim's original proposal](https://docs.google.com/document/d/12xAkRcSq9hZVEpcO56EIiEYmd0ivybWo4YRXV0Nfq-8)\n\n- [Mike Spreitzer's first proposal](https://docs.google.com/document/d/1YW_rYH6tvW0fvny5b7yEZXvwDZ1-qtA-uMNnHW9gNpQ)\n\n- [Daniel Smith's proposal](https://docs.google.com/document/d/1BtwFyB6G3JgYOaTxPjQD-tKHXaIYTByKlwlqLE0RUrk)\n\n- [Mike's second proposal](https://docs.google.com/document/d/1c5SkLHvA4H25sY0lihJtu5ESHm36h786vi9LaQ8xdtY)\n\n- [Min's second proposal](https://github.com/kubernetes/enhancements/pull/864)\n\n- [Daniel's brain dump](https://docs.google.com/document/d/1cwNqMDeJ_prthk_pOS17YkTS_54D8PbFj_XwfJNe4mE)\n\n- [Mike's third proposal](https://github.com/kubernetes/enhancements/pull/930)\n\n- [Mike's proposed first cut](https://github.com/kubernetes/enhancements/pull/933)\n\nAlso notable are the following notes from meetings on this subject.\n\n- https://docs.google.com/document/d/1bEh2BqfSSr3jyh1isnXDdmfe6koKd_kMXCFj08uldf8\n- https://docs.google.com/document/d/1P8NRaQaJBiBAP2Bb4qyunJpyQ-4JVzripfCi3UXG9zc\n\n### Design Considerations\n\nFollowing is an attempt to summarize the issues addressed in those\nproposals and the current thinking on them; the current proposal\nattempts to respond.\n\nDespite the open issues, we seem to be roughly agreed on an outline\nsomething like the following.\n\n- When a request arrives at the handler, the request is categorized\n  somehow.  The nature of the categories and the categorization\n  process is one open issue. Some proposals allow for the request to\n  be rejected upon arrival based on that categorization and some local\n  state.  Unless rejected, the request is put into a FIFO queue.  That\n  is one of many queues.  The queues are associated with the\n  categories somehow.  Some proposals contemplate ejecting less\n  desirable requests to make room for the newly queued request, if and\n  when queue space is tight.\n\n- A request might also be rejected at a later time, based on other\n  reject a request at the head of the queue if the latency is found to\n  be too big at certain times.\n\n- Based on some resource limit (e.g., QPS or concurrency) and with\n  regards to priority and fairness criteria, requests are dispatched\n  from the queues to be served (i.e., continue down the handler\n  chain).\n\n- We assume that when the requst-timeout handler aborts a request it\n  memory at that point.  We know that this is not actually true today,\n  but is intended; we leave fixing that to be independent work, and\n  for now this KEP simply ignores the gap.\n\nOne of the biggest questions is how to formulate the scheduling\nparameters.  There are several related concepts in the state of the\nart of scheduling, and we are trying to figure out what to adopt\nand/or invent.  We would prefer to invent as little as possible; it is\nscheduling algorithm and prove it correct.  The vmware product line\nuses scheduling parameters named \"reservation\", \"limit\", and \"shares\"\nfor each class of workload.  The first two are known elsewhere as\n\"assured\" and \"ceiling\".  Finding a published algorithm that\nimplements all three has not proven easy.  Alternatively, priorities\nis some form of fairness within each priority level.  The current\nthinking is in that direction: use priorities, with simple equal\nfairness among some categories of requests in each priority level.\nThere are published scheduling algorithms that provide fairness, and\nwe hope to use/adapt one of them to apply independently within the\nconfines of each priroity level.\n\nAnother issue is whether to manage QPS or concurrency or what.\nManaging QPS leaps first to mind, perhaps because it is a simple\nconcept and perhaps because it is familiar from the self-restraint\nthat clients apply today.  But we want to also take service time into\naccount; a request flow with longer service times should get less QPS\napiserver.  A natural way to do this is with an inverse linear\nrelation.  For example, when two CPU-bound request flows are getting\nequal CPU from the apiserver, and the first flow's requests have a\nservice time that is X times the service time of the second flow's\nrequests, the first flow's QPS is 1/X of the second's.  This is\nexactly analogous to what happens in networking: if two flows are\ngetting the same bandwidth, and one flow's packets are X times as big\nas the second's, then the first flow's packets per second rate is 1/X\nthat of the second flow.  This inverse linear relation amounts to\nmanaging the product of QPS * service time.  That is equivalent to\nmanaging concurrency.  Managing concurrency is an obvious choice for\nmemory, and we now see it is a good choice for CPU too.  This is also\na convenient choice because it is what the max-in-flight handler is\ndoing today, so we would be making a relatively modest extension to\nthat handler's conception.\n\nCompared to traditional scheduling problems, ours is harder because of\nthe combination of these facts: (1) (unlike a router handling a\npacket) the apiserver does not know beforehand how long a request will\ntake to serve nor how much memory it will consume, and (2) (unlike a\nCPU scheduler) the apiserver can not suspend and resume requests.\nAlso, we are generally loathe to abort a request once it has started\nbeing served.  We may some day consider doing this for low-priority\nlong-running requests, but are not addressing long-running requests at\nfirst.  We are leaning towards adapting well known and studied\nscheduling technique(s); but adaptation is a form of invention, and we\nhave not converged yet on what to do here.\n\nAnother issue is how to combine two goals: protection of CPU, and\nprotection of memory.  A related issue is the fact that there are two\nstages of memory consumption: a request held in a queue holds some\nmemory, and a request being served may use a lot more.  The current\nthinking seems to be focusing on using one QPS or concurrency limit on\nrequests being served, on the expectation that this limit can be set\nto a value that provides reasonable protection for both CPU and memory\nwithout being too low for either.\n\nIf we only limmit requests being served then the queues could cause\ntwo problems: consuming a lot of apiserver memory, and introducing a\nlot of latency.  For the latter we are aware of some solutions from\nthe world of networking, [CoDel](https://en.wikipedia.org/wiki/CoDel)\nand\n[fq_codel](https://tools.ietf.org/html/draft-ietf-aqm-fq-codel-06).\nCoDel is a technique for ejecting requests from a queue for the\npurpose of keeping latency low, and fq_codel applies the CoDel\ntechnique in each of many queues.  CoDel is explicitly designed to\nwork in the context of TCP flows on the Internet.  This KEP should be\nsimilarly explicit about the context, particularly including what is\nthe feedback given to clients and how do they react and what is the\nnet effect of all the interacting pieces.  No such analysis has yet\nbeen done for any of the proposals.\n\nThe CoDel technique is described as parameterless but has two magic\nnumbers: an \"initial interval\" of 100 ms and a \"target\" of 5 ms.  The\ninitial interval is set based on round trip times in the Internet, and\nthe target is set based on a desired limit on the latency at each hop.\nWhat are the analogous numbers for our scenario?  We do not have large\nnumbers of hops; typically at most two (client to main apiserver and\nthen main apiserver to aggregated apiserver).  What is analogous to\nnetwork round trip time?  We have a latency goal of 1 second, and a\nrequest service time limit of 1 minute.  If we take 1 second as the\ninitial interval then note that the maximum service time is much\nlarger than the initial interval; by contrast, in networking, the\nmaximum service time (i.e., packet length / link speed) is much\nsmaller than the initial interval.  Even if we take 1 minute as our\ninitial interval, we still do not have the sort of relationship that\nobtains in networking.  Note that in order to get good statistics on a\nmany requests served during an interval.  Because of this mismatch,\nand because equivalence of context has not been established, we are\nnot agreed that the CoDel technique can be used.\n\nNote that the resource limit being applied is a distinct concept from\nthe fairness criteria.  For example, in CPU scheduling there may be 4\nCPUs and 50 threads being scheduled onto those CPUs; we do not suppose\nthe goal is to have each thread to be using 0.08 CPUs at each instant;\na thread uses either 0 or 1 CPUs at a given instant.  Similarly, in\nnetworking, a router may multiplex a thousand flows onto one link; the\ngoal is not to have each flow use 1/1000th of the link at each\ninstant; a packet uses 0 links while queued and 1 link while being\ntransmitted.  Each CPU or link is used for just one thing at a time;\nthis is the resource limit.  The fairness goal is about utilization\nobserved over time.  So it is in our scenario too.  For example, we\nmay have 5000 flows of requests and a concurrency limit of 600\nrequests at any one time.  That does not mean that our goal is for\neach flow to have 0.12 requests running at each instant.  Our goal is\nto limit the number of running requests to 600 at each instant and\nprovide some fairness in utilization averaged over time.\n\nThat average over time must not be over too much or too little time.\nIt would not make sense to average over all past time; that would\nallow a flow to build up a huge amount of credit, enabling it to crowd\nout other flows.  It also does not make sense for the average to cover\na small amount of time.  Because serving requests, like transmitting\npackets, is lumpy we must average over many service times.  Approaches\nto this include: using a sequence of intervals, using a sliding\nwindow, and using an exponential decay.\n\nAnother open issue is the categorization: what are the categories and\nhow is a request assigned to a category?  We seem to be agreed on at\nleast one important point: each request is assigned to exactly one\ncategory, and is handled by exactly one \"bucket\" or \"queue\".  We also\nseem to be converging toward a two-level hierarchy of categories,\naligned with the handling outline discussed earlier: at the higher\nlevel there are priorities, and within each priority level there is a\ncollection of flows that compete fairly.\n\nIt is desired to allow lesser priority traffic to get some service\neven while higher priority traffic is arriving at a sufficient rate to\nentirely occupy the server.  There needs to be a quantitative\ndefinition of this relation between the priorities, and an\nimplementation that (at least roughly) implements the desired\nquantitative relation.\n\npriority associations.  The predicate can test any authenticated\nwork being requested.  One issue to nail down is the question of what\nhappens if multiple predicates match a given request; the handler\nshould pick exactly one priority level for each request.\n\nWithin a given priority level we want to give a fair division of\ncapacity among several \"flows\"; the lower level of categorization is\nhow to compute a flow identifier from a request.\n\nThe handler may additionally hash the flows into queues, so that a\nmore manageable number of queues is involved.  Shuffle sharding can be\nused to make it more likely that a mouse can avoid being squashed by\nan elephant.\n\nSome of the proposals draw inspiration from deficit (weighted or not)\nround robin, and some from (weighted or not) fair queuing.  DRR has\nquantum is larger than the largest packet.  In our case that would be\nquite large indeed, since the timeout for a request is typically 1\nminute (it would be even worse if we wanted to handle WATCH requests).\nThe dispatching of the DRR technique is bursty, and the size of the\nburst increases with the quantum.  The proposals based on DRR tend to\ngo with a small quantum, presumably to combat the burstiness.  The\nlogical extreme of this, in the unweighted case, is to use a quantum\nof 1 bit (in the terms of the original networking setting).  That is\nisomorphic to (unweighted) fair queuing!  The weighted versions, still\nwith miniscule quanta, are also isomorphic.\n\n### Test Plan\n\n- __Unit Tests__: All changes must be covered by unit tests. Additionally,\n we need to test the evenness of dispatching algorithm.\n- __Integration Tests__: The use cases discussed in this KEP must be covered by integration tests.\n\n### Graduation Criteria\n\nAlpha:\n\n- Necessary defaulting, validation\n- Adequate documentation for the changes\n- Minimum viable test cases mentioned in Test Plan section\n\n## Implementation History\n\n(none yet)\n\n## Drawbacks\n\nIncreased burden on operator to provide good configuration.\n\nIncrease in stuff to consider when analyzing performance results.\n\nIncreased complexity of code to maintain.\n\nIncreased runtime costs.\n\n## Alternatives\n\nOnce we have settled on a design there will be things to say about the\ndesigns not chosen.\n\n## Infrastructure Needed\n\nThe end-to-end test suite should exercise the functionality introduced\nby this KEP.  This may require creating a special client to submit an\noverload of low-priority work.\n"
+  },
+  {
+    "id": "4081034b234908cf234cb7019c4d9d6a",
+    "title": "Graduate Server-side Get and Partial Objects to GA",
+    "authors": ["@smarterclayton"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["sig-cli"],
+    "reviewers": ["@lavalamp", "@soltysh", "@liggitt"],
+    "approvers": ["@liggitt", "@pwittrock"],
+    "editor": "TBD",
+    "creationDate": "2019-03-22",
+    "lastUpdated": "2019-03-22",
+    "status": "implementable",
+    "seeAlso": [
+      "https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/server-get.md"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Graduate Server-side Get and Partial Objects to GA\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [1.15](#115)\n  - [1.16](#116)\n  - [1.19](#119)\n  - [Implementation Details](#implementation-details)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nServer-side columnar formatting and partial object metadata has been in beta since Kube 1.10 and as of 1.15 is consistently implemented and in wide use as part of `kubectl` and other web interfaces. This document outline required steps to graduate it to GA. \n\n## Motivation\n\nThe user of server-side printing with CRDs and Aggregated APIs is common and is a key part of providing equivalence between built-in and extension APIs for administrator usability. We have not needed to update the schema since 1.10, proving that it is ready to ship. Promoting it to GA in 1.15 will allow us to declare the kubectl portion feature complete and remove the legacy printers, and also update controllers that would benefit from use of PartialObjectMetadata (garbage collector, namespace controller, and quota counter) without fear of deprecation. PartialObjectMetadata allows these controllers to perform protobuf list actions efficiently when retrieving only the core object metadata.\n\n### Goals\n\nServer-side printing has no outstanding feature requests now that full [WATCH support has been implemented in 1.15](https://github.com/kubernetes/kubernetes/pull/71548). It is ready to move to GA by promoting the resources.\n\nPartialObjectMetadata exposes our full ObjectMeta interface and no API changes are anticipated. However, to prove their value one of the dynamic controllers should be ported in 1.15 to use PartialObjectMetadata instead of Unstructured objects to demonstrate the gains in performance. If successful PartialObjectMetadata would also be candidate for GA in 1.15.\n\n### Non-Goals\n\n* Changes to the `Table` object not directly concerned with supporting `kubectl` or other consumers\n* Changes to `PartialObjectMetadata` that are not related to backend implementation.\n\n## Proposal\n\n### 1.15\n\n* Copy `Table`, `PartialObjectMetadata` and `PartialObjectMetadataList` to `meta/v1` and expose the transformations in the API server.\n  * Update the serialization of `PartialObjectMetadataList` to use protobuf id `1` for `ListMeta` (it was added late in v1beta1)\n\n### 1.16\n\n* Update controllers to use `PartialObjectMetadata` `v1`.\n  * In the garbage collector, we will remove the need to call `Update` and use a partial object metadata client/informer\n  * In the namespace controller, we will use a partial object metadata informer\n  * In the quota counting code, we will use a partial object metadata informer\n* Announce deprecation of `v1beta1` objects and removal in 1.19 \n* `kubectl` should switch to using `meta.k8s.io/v1` `Table` (supporting 1.15+ clusters)\n\n### 1.19\n\n* Remove `meta.k8s.io/v1beta1`\n\n### Implementation Details\n\nA new dynamic client variant capable of supporting read and write operations on PartialObjectMetadata\nshould be created that hides whether the server supports PartialObjectMetadata. \n\nCurrently `v1beta1.Table` does not support Protobuf and the generators do not trivially support the\nserialization of the cells. We need to decide on a serialization format for the Protobuf cells and\nensure generators can be made to support it. This work does not need to block `v1` for Protobuf\nbecause the clients that access `Tables` are almost exclusively JSON clients (web consoles and CLIs).\n`PartialObjectMetadata*` will have a full protobuf implementation.\n\n### Risks and Mitigations\n\nThe primary risk is that we identify a `v1beta1.Table` schema change post freeze. Wider use might\ngather that feedback, but the schema is deliberately left simple to allow us to grow in the future.\n\n## Graduation Criteria\n\nThe following code changes must be made to take `Table` GA\n\n* Move API objects to `v1` and support conversion internally\n* Update REST infra to support transforming objects at that version\n\nThe following code changes must be made to take `PartialObjectMetadata` GA\n\n* Move API objects to `v1` and support conversion internally\n* Update REST infra to support transforming objects at that version\n\nThe following code changes should be made before `PartialObjectMetadata` is GA to get feedback\n\n* Update all of (GC, Namespace, Quota counting) to use a new PartialObjectMetadata specific typed client using protobuf.\n\n### Version Skew Strategy\n\nWe will support N-1 for both `kubectl` and participating controllers by introducing GA first and then deprecating and removing v1beta1 in the subsequent release. There are no serialized forms of these objects and so on-disk format is not a concern.\n\n## Implementation History\n\n* First version of this proposal merged.\n* Server-side GET objects moved to v1 in 1.15\n"
+  },
+  {
+    "id": "8e49872c85d384fbd462b52b8900da10",
+    "title": "Union types",
+    "authors": ["@apelisse"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": null,
+    "reviewers": ["@sttts", "@lavalamp", "@thockin", "@DirectXMan12"],
+    "approvers": ["@lavalamp"],
+    "editor": "TBD",
+    "creationDate": "2019-03-25",
+    "lastUpdated": "2019-03-25",
+    "status": "implementable",
+    "seeAlso": ["/keps/sig-api-machinery/0006-apply.md"],
+    "replaces": ["https://docs.google.com/document/d/1lrV-P25ZTWukixE9ZWyvchfFR0NE2eCHlObiCUgNQGQ"],
+    "supersededBy": null,
+    "markdown": "\n# Union types\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Go tags](#go-tags)\n  - [OpenAPI](#openapi)\n  - [Discriminator](#discriminator)\n  - [Normalizing on updates](#normalizing-on-updates)\n    - [\u0026quot;At most one\u0026quot; versus \u0026quot;exactly one\u0026quot;](#at-most-one-versus-exactly-one)\n    - [Clearing all the fields](#clearing-all-the-fields)\n  - [Backward compatibilities properties](#backward-compatibilities-properties)\n  - [Validation](#validation)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n- [Future Work](#future-work)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [x] KEP approvers have set the KEP status to `implementable`\n- [x] Design details are appropriately documented\n- [x] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [x] Graduation criteria is in place\n- [x] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nModern data model definitions like OpenAPI v3 and protobuf (versions 2 and 3)\nhave a keyword to implement “oneof” or \"union\". They allow APIs to have a better\nsemantics, typically as a way to say “only one of the given fields can be\nset”. We currently have multiple occurrences of this semantics in kubernetes core\ntypes, at least:\n- VolumeSource is a structure that holds the definition of all the possible\n  volume types, only one of them must be set, it doesn't have a discriminator.\n- DeploymentStrategy is a structure that has a discrminator\n  \"DeploymentStrategyType\" which decides if \"RollingUpate\" should be set\n\nThe problem with the lack of solution is that:\n- The API is implicit, and people don't know how to use it\n- Clients can't know how to deal with that, especially if they can't parse the\n  OpenAPI\n- Server can't understand the user intent and normalize the object properly\n\n## Motivation\n\nCurrently, changing a value in an oneof type is difficult because the semantics\nis implicit, which means that nothing can be built to automatically fix unions,\nleading to many bugs and issues:\n- https://github.com/kubernetes/kubernetes/issues/35345\n- https://github.com/kubernetes/kubernetes/issues/24238\n- https://github.com/kubernetes/kubernetes/issues/34292\n- https://github.com/kubernetes/kubernetes/issues/6979\n- https://github.com/kubernetes/kubernetes/issues/33766\n- https://github.com/kubernetes/kubernetes/issues/24198\n- https://github.com/kubernetes/kubernetes/issues/60340\n\nAnd then, for other people:\n- https://github.com/rancher/rancher/issues/13584\n- https://github.com/helm/charts/pull/12319\n- https://github.com/EnMasseProject/enmasse/pull/1974\n- https://github.com/helm/charts/pull/11546\n- https://github.com/kubernetes/kubernetes/pull/35343\n\nThis is replacing a lot of previous work and long-standing effort:\n- Initially: https://github.com/kubernetes/community/issues/229, then\n- https://github.com/kubernetes/community/pull/278\n- https://github.com/kubernetes/community/pull/620\n- https://github.com/kubernetes/kubernetes/pull/44597\n- https://github.com/kubernetes/kubernetes/pull/50296\n- https://github.com/kubernetes/kubernetes/pull/70436\n\nServer-side [apply](http://features.k8s.io/555) is what enables this proposal to\nbecome possible.\n\n### Goals\n\nThe goal is to enable a union or \"oneof\" semantics in Kubernetes types, both for\nin-tree types and for CRDs.\n\n### Non-Goals\n\nWe're not planning to use this KEP to release the feature, but mostly as a way\nto document what we're doing.\n\n## Proposal\n\nIn order to support unions in a backward compatible way in kubernetes, we're\nproposing the following changes.\n\nNote that this proposes unions to be \"at most one of\". Whether exactly one is\nsupported or not should be implemented by the validation logic.\n\n### Go tags\n\nWe're proposing a new type of tags for go types (in-tree types, and also\nkubebuilder types):\n\n- `// +union` before a structure means that the structure is a union. All the\n  fields must be optional (beside the discriminator) and will be inlucded as\n  members of the union. That structure CAN be embedded in another structure.\n- `// +unionDeprecated` before a field means that it is part of the\n  union. Multiple fields can have this prefix. These fields MUST BE optional,\n  omitempty and pointers. The field is named deprecated because we don't want\n  people to embed their unions directly in structures, and only exist because of\n  some existing core types (e.g. `Value` and `ValueFrom` in\n  [EnvVar](https://github.com/kubernetes/kubernetes/blob/3ebb8ddd8a21b/staging/src/k8s.io/api/core/v1/types.go#L1817-L1836)).\n- `// +unionDiscriminator` before a field means that this field is the\n  discriminator for the union. Only one field per structure can have this\n  prefix. This field HAS TO be a string, and CAN be optional.\n\nMultiple unions can exist per structure, but unions can't span across multiple\ngo structures (all the fields that are part of a union has to be together in the\nsame structure), examples of what is allowed:\n\n```\n// This will have one embedded union.\ntype TopLevelUnion struct {\n\tName string `json:\"name\"`\n\n\tUnion `json:\",inline\"`\n}\n\n// This will generate one union, with two fields and a discriminator.\n// +union\ntype Union struct {\n\t// +unionDiscriminator\n\t// +optional\n\tUnionType string `json:\"unionType\"`\n\n\t// +optional\n\tFieldA int `json:\"fieldA\"`\n    // +optional\n\tFieldB int `json:\"fieldB\"`\n}\n\n// This also generates one union, with two fields and on discriminator.\ntype Union2 struct {\n\t// +unionDiscriminator\n\tType string `json:\"type\"`\n\t// +unionDeprecated\n    // +optional\n\tAlpha int `json:\"alpha\"`\n\t// +unionDeprecated\n    // +optional\n\tBeta int `json:\"beta\"`\n}\n\n// This has 3 embedded unions:\n// One for the fields that are directly embedded, one for Union, and one for Union2.\ntype InlinedUnion struct {\n\tName string `json:\"name\"`\n\n\t// +unionDeprecated\n\t// +optional\n\tField1 *int `json:\"field1,omitempty\"`\n\t// +unionDeprecated\n\t// +optional\n\tField2 *int `json:\"field2,omitempty\"`\n\n\tUnion  `json:\",inline\"`\n\tUnion2 `json:\",inline\"`\n}\n```\n\n### OpenAPI\n\nOpenAPI v3 already allows a \"oneOf\" form, which is accepted by CRD validation\n(and will continue to be accepted in the future). That oneOf form will be used\nfor validation, but is \"on-top\" of this proposal.\n\nA new extension is created in the openapi to describe the behavior:\n`x-kubernetes-unions`.\n\nThis is a list of unions that are part of this structure/object. Here is what\neach list item is made of:\n- `discriminator: \u003cdiscriminator\u003e` is set to the name of the discriminator\n  field, if present,\n- `fields-to-discriminateBy: {\"\u003cfieldName\u003e\": \"\u003cdiscriminateName\u003e\"}` is a map of\n  fields that belong to the union to their discriminated names. The\n  discriminatedValue will typically be set to the name of the Go variable.\n\nConversion between OpenAPI v2 and OpenAPI v3 will preserve these fields.\n\n### Discriminator\n\nFor backward compatibility reasons, discriminators should be added to existing\nunion structures as an optional string. This has a nice property that it's going\nto allow conflict detection when the selected union field is changed.\n\nWe also do strongly recommend new APIs to be written with a discriminator, and\ntools (kube-builder) should probably enforce the presence of a discriminator in\nCRDs.\n\nThe value of the discriminator is going to be set automatically by the apiserver\nwhen a new field is changed in the union. It will be set to the value of the\n`fields-to-discriminateBy` for that specific field.\n\nWhen the value of the discriminator is explicitly changed by the client, it\nwill be interpreted as an intention to clear all the other fields. See section\nbelow.\n\n### Normalizing on updates\n\nA \"normalization process\" will run automatically for all creations and\nmodifications (either with update or patch). It will happen automatically in order\nto clear fields and update discriminators. This process will run for both\ncore-types and CRDs. It will take place before validation. The sent object\ndoesn't have to be valid, but fields will be cleared in order to make it valid.\nThis process will also happen before fields tracking (server-side apply), so\nchanges in discriminator, even if implicit, will be owned by the client making\nthe update (and may result in conflicts).\n\nThis process works as follows:\n- If there is a discriminator, and its value has changed, clear all fields but\n  the one specified by the discriminator,\n- If there is no discriminator, or if its value hasn't changed,\n  - if there is exactly one field, set the discriminator when there is one\n    to that value. Otherwise,\n  - compare the fields set before and after. If there is exactly one field\n    added, set the discriminator (if present) to that value, and remove all\n    other fields. if more than one field has been added, leave the process so\n    that validation will fail.\n\n#### \"At most one\" versus \"exactly one\"\n\nThe goal of this proposal is not to change the validation, but to help clients\nto clear other fields in the union. Validation should be implemented for in-tree\ntypes as it is today, or through \"oneOf\" properties in CRDs.\n\nIn other word, this is proposing to implement \"at most one\", and the exactly one\nshould be provided through another layer of validation (separated).\n\n#### Clearing all the fields\n\nSince the system is trying to do the right thing, it can be hard to \"clear\neverything\". In that case, each API could decide to have their own \"Nothing\"\nvalue in the discriminator, which will automatically trigger a clearing of all\nfields beside \"Nothing\".\n\n### Backward compatibilities properties\n\nThis normalization process has a few nice properties, especially for dumb\nclients, when it comes to backward compatibility:\n- A dumb client that doesn't know which fields belong to the union can just set\n  a new field and get all the others cleared automatically\n- A dumb client that doesn't know about the discriminator is going to change a\n  field, leave the discriminator as it is, and should still expect the fields to\n  be cleared accordingly\n- A dumb client that knows about the discriminator can change the discriminator\n  without knowing which fields to clear, they will get cleared automatically\n\n\n### Validation\n\nObjects have to be validated AFTER the normalization process, which is going to\nleave multiple fields of the union if it can't normalize. As discussed in\ndrawbacks below, it can also be useful to validate apply requests before\napplying them.\n\n### Test Plan\n\nThere are mostly 3 aspects to this plan:\n- [x] Core functionality, isolated from all other components: https://github.com/kubernetes-sigs/structured-merge-diff/blob/master/typed/union_test.go\n- [x] Functionality as part of server-side apply: How human and robot interactions work: https://github.com/kubernetes-sigs/structured-merge-diff/blob/master/typed/union_test.go\n- [x] Integration in kubernetes: https://github.com/kubernetes/kubernetes/pull/77370/files#diff-4ac5831d494b1b52c7c7be81e552a458\n\n### Graduation Criteria\n\nSince this project is a sub-project of Server-side apply, it will be introduced\ndirectly as Beta, and will graduate to GA in a later release, according to the\ncriteria below.\n\n#### Beta -\u003e GA Graduation\n\n- CRD support has been proven successful\n- Core-types all implement the semantics properly\n- Stable and bug-free for two releases\n\n## Future Work\n\nSince the proposal only normalizes the object after the patch has been applied,\nit is hard to fail if the patch is invalid. There are scenarios where the patch\nis invalid but it results in an unpredictable object. For example, if a patch\nsends both a discriminator and a field that is not the discriminated field, it\nwill either clear the value sent if the discriminator changes, or it will change\nthe value of the sent discriminator.\n\nValidating patches is not a problem that we want to tackle now, but we can\nvalidate \"Apply\" objects to make sure that they do not define such broken\nsemantics.\n\n## Implementation History\n\nHere are the major milestones for this KEP:\n- Initial discussion happened a year before the creation of this kep:\n  https://docs.google.com/document/d/1lrV-P25ZTWukixE9ZWyvchfFR0NE2eCHlObiCUgNQGQ/edit#heading=h.w5eqnf1f76x5\n- Points made in the initial document have been improved and put into this kep,\n  which has approved by sig-api-machinery tech-leads\n- KEP has been implemented:\n  - logic mostly lives in sigs.k8s.io/structured-merge-diff\n  - conversion between schema and openapi definition are in k8s.io/kube-openapi\n  - core types have been modified in k8s.io/kubernetes\n- Feature is ready to be released in Beta in kubernetes 1.15\n"
+  },
+  {
+    "id": "bd55d64cec1ea5f76cd8c9e207aeeaa6",
+    "title": "Less object serializations",
+    "authors": ["@wojtek-t"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["sig-scalability"],
+    "reviewers": ["@jpbetz", "@justinsb", "@smarterclayton"],
+    "approvers": ["@deads2k", "@lavalamp"],
+    "editor": "",
+    "creationDate": "2019-03-27",
+    "lastUpdated": "2019-10-10",
+    "status": "implemented",
+    "seeAlso": ["TODO"],
+    "replaces": ["n/a"],
+    "supersededBy": ["n/a"],
+    "markdown": "\n# Less object serializations\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n  - [Bake-in caching objects into apimachinery](#bake-in-caching-objects-into-apimachinery)\n  - [LRU cache](#lru-cache)\n  - [Smart objects](#smart-objects)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n**ACTION REQUIRED:** In order to merge code into a release, there must be an issue in [kubernetes/enhancements] referencing this KEP and targeting a release milestone **before [Enhancement Freeze](https://github.com/kubernetes/sig-release/tree/master/releases)\nof the targeted release**.\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e., [kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be updated for the enhancement to be released.\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nScalability and performance of kube-apiserver is crucial for scalability\nof the whole Kubernetes cluster. Given that kube-apiserver is cpu-intensive\nprocess, scaling a single instance of it translates to optimizing amount\nof work is needed to process a request (cpu cycles and amount of allocated\nmemory, as memory management is significant part of work done be\nkube-apiserver).\n\nThis proposal is aiming to significantly reduce amount of work spent on\nserializing objects as well as amount of allocated memory to process that.\n\n## Motivation\n\nRunning different types of scalability tests and analyzing large production\nclusters proves that large number of watches watching the same set of objects\nmay cause significant load on kube-apiserver. An extreme example of it is\n[#75294][], where creation of a single large Endpoints object (almost 1MB of\nsize, due to 5k pods backing it) in 5k-node cluster can completely overload\nkube-apiserver for 5 seconds.\n\nThe main reason for that is that for every watcher (Endpoints are being watched\nby kube-proxy running on every one) kube-apiserver independently serializes\n(which also requires deep-copy) every single object being send via this watch.\n\nWhile this problem is extremely visible for watch, the situation looks the same\nfor regular GET/LIST operations - reading the same object N times will result\nin serializing that N times independently.\n\nThis proposal presents a solution for that problem.\n\n[#75294]: https://github.com/kubernetes/kubernetes/issues/75294\n\n### Goals\n\n- Reduce load on kube-apiserver and number of memory allocations, by avoiding\nserializing the same object multiple times for different watchers.\n\n### Non-Goals\n\n- Change overall architecture of the system, by changing what data is being\nread/watched by different components.\n\n## Proposal\n\nThis proposal does not introduce any user-visible changes - the proposed changes\nare purely implementation details of kube-apiserver.\n\nFirst, we will extend [runtime.Encoder][] interface with the Identifier() method:\n```go\n// Identifier represents an identifier.\n// Identitier of two different objects should be equal if and only if for every\n// input the output they produce is exactly the same.\ntype Identifier string\n\ntype Encoder interface {\n\t...\n\t// Identifier returns an identifier of the encoder.\n\t// Identifiers of two different encoders should be equal if and only if for every input\n\t// object it will be encoded to the same representation by both of them.\n\tIdentifier() Identifier\n}\n```\n\nWith that, we will introduce a new interface:\n```go\n// CacheableObject allows an object to cache its different serializations\n// to avoid performing the same serialization multiple times.\ntype CacheableObject interface {\n\t// CacheEncode writes an object to a stream. The \u003cencode\u003e function will\n\t// be used in case of cache miss. The \u003cencode\u003e function takes ownership\n\t// of the object.\n\t// If CacheableObject is a wrapper, then deep-copy of the wrapped object\n\t// should be passed to \u003cencode\u003e function.\n\t// CacheEncode assumes that for two different calls with the same \u003cid\u003e,\n\t// \u003cencode\u003e function will also be the same.\n\tCacheEncode(id Identifier, encode func(Object, io.Writer) error, w io.Writer) error\n\n\t// GetObject returns a deep-copy of an object to be encoded - the caller of\n\t// GetObject() is the owner of returned object. The reason for making a copy\n\t// is to avoid bugs, where caller modifies the object and forgets to copy it,\n\t// thus modifying the object for everyone.\n\t// The object returned by GetObject should be the same as the one that is supposed\n\t// to be passed to \u003cencode\u003e function in CacheEncode method.\n\t// If CacheableObject is a wrapper, the copy of wrapped object should be returned.\n\tGetObject() Object\n```\n\nWe will add support for `CacheableObject` for all existing Encoders. This is\nbasically as simple as:\n```go\nfunc (e *Encoder) Encode(obj Object, stream io.Writer) error {\n\tif co, ok := obj.(CacheableObject); ok {\n\t\treturn co.CacheEncode(s.Identifier(), s.doEncode, stream)\n\t}\n\treturn s.doEncode(obj, stream)\n}\n\nfunc (e *Encoder) doEncode(obj Object, stream io.Writer) error {\n\t// Existing encoder logic.\n}\n```\n\nNecessary generic tests will be created to ensure it is supported correctly.\n\n[runtime.Encoder]: https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/runtime/interfaces.go#L52\n\nWith those (relatively mechanical) changes, we will introduce an internal type\nin package `cacher` implementing both `runtime.Object` and `CacheableObject`\ninterfaces. The idea behind it is that it will be encapsulating the original\nobject and additionally it will be able to accumulate its serialized versions.\nIt will look like this:\n```go\n// serializationResult captures a result of serialization.\ntype serializationResult struct {\n\t// once should be used to ensure serialization is computed once.\n\tonce sync.Once\n\n\t// raw is serialized object.\n\traw []byte\n\t// err is error from serialization.\n\terr error\n}\n\n// metaRuntimeInterface implements runtime.Object and\n// metav1.Object interfaces.\ntype metaRuntimeInterface interface {\n\truntime.Object\n\tmetav1.Object\n}\n\n// cachingObject is an object that is able to cache its serializations\n// so that each of those is computed exactly once.\n//\n// cachingObject implements the metav1.Object interface (accessors for\n// all metadata fields). However, setters for all fields except from\n// SelfLink (which is set lately in the path) are ignored.\ntype cachingObject struct {\n\tlock sync.RWMutex\n\n\t// Object for which serializations are cached.\n\tobject metaRuntimeInterface\n\n\t// serializations is a cache containing object`s serializations.\n\t// The value stored in atomic.Value is of type serializationsCache.\n\t// The atomic.Value type is used to allow fast-path.\n\tserializations atomic.Value\n}\n```\n\nIn the initial attempt, watchCache when receiving an event via watch from\netcd will be opaquing it into `CachingObject` and operating on object of\nthat type later.\n\nThat means that we won't have gains from avoid serialization for any GET/LIST\nrequests server from cache as well as for `init event` that we process when\ninitializing a new watch, but that seems good enough for the initial attempt.\nThe obvious gain from it is that the memory used for caching is used only\nfor a very short period of time (when delivering this watch to watchers) and\nquickly released, which means we don't need to be afraid about increased\nmemory usage.\nWe may want to revisit that decision later if we would need more gains\nfrom avoiding serialization and deep-copies of objects in watchcache.\n\nBased on the implementation, we observed the following gains:\n- eliminating kube-apiserver unresponsiveness in case of write of\na single huge Endpoints object: [#75294#comment-472728088][]\n- ~5% lower cpu-usage\n- ~15% less memory allocations\n\n[#75294#comment-472728088]: https://github.com/kubernetes/kubernetes/issues/75294#issuecomment-472728088\n\n### Risks and Mitigations\n\nThe proposal doesn't introduce any user visible change - the only risk is\nrelated to bugs in implementation. Even though, the serialization code is\nwidely user by all end-to-end tests and bugs should be catched by those\nor unit tests of newly added logic, we will try to mitigate the risk by\nintroducing a feature gate and hiding the logic of using the newly introduced\nobject behind this feature gate.\n\n## Design Details\n\n### Test Plan\n\n* Unit tests covering all corner cases of logic of newly introduced objects.\n* Unit test to detect races of newly introduced objects\n* Regular e2e tests are passing.\n\n### Graduation Criteria\n\n* All existing e2e tests are passing.\n* Scalability tests confirm gains of that change.\n\nWe're planning to enable this feature by default, but a feature gate to\ndisable it is the mitigation strategy if bugs will be discovered after\nrelease.\n\n### Upgrade / Downgrade Strategy\n\nThis feature doesn't change any persistent state of the cluster, just the\nin-memory representation of objects, upgrade/downgrade strategy is not\nrelevant to this feature.\n\n### Version Skew Strategy\n\nThe feature is only changing in-memory representation of objects only in\nkube-apiserver, so version skew strategy is not relevant.\n\n## Implementation History\n\n- 2019-03-27: KEP Created\n- 2019-07-18: KEP Merged\n- 2019-07-19: KEP updated with test plan and moved to implementaable state.\n- 2019-10-10: KEP updated to reflect the implementation.\n- v1.17: Implemented\n\n## Alternatives\n\n### Bake-in caching objects into apimachinery\nWe considered making objects above part of apimachinery.\n\nPros:\n- Expose ability to use it for others\n\nCons:\n- Complicated code hits apimachinery\n\n### LRU cache\nWe considered using simple LRU cache to store serialized objects.\n\nPros:\n- performance gains also for reads served from etcd (though these\ndoesn't seem to be huge based on experiments)\n\nCons:\n- potentially significant point of contention\n- no-control over what is still cached (e.g. for frequently changing\nresources, we still keep them in cache, even if they will never be\nserved again)\n\n### Smart objects\nWe also considered using `smart objects` - an object that carries the\nserialized format of object from etcd with itself.\n\nPros:\n- very clear encapsulation\n\nCons:\n- We need an ability to in-place add fields to serialized object\n(i.e. SelfLink) - very tricky and error-prone\n- This doesn't work across different (group, version) pairs. As\nan example, if at some point we will be migrating `Endpoints`\nobject to the new API, this will stop working for the whole\nmigration period (i.e. at least one release).\n"
+  },
+  {
+    "id": "39ef6f610905a731d9e0c30a1864a6c8",
+    "title": "CustomResourceDefinition Conversion Webhook",
+    "authors": ["@mbohlool", "@erictune"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": null,
+    "reviewers": ["@lavalamp", "@deads2k", "@sttts", "@liggitt"],
+    "approvers": ["@lavalamp", "@deads2k"],
+    "editor": "",
+    "creationDate": "2019-04-25",
+    "lastUpdated": "2019-04-25",
+    "status": "implemented",
+    "seeAlso": null,
+    "replaces": [
+      "(https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/customresource-conversion-webhook.md)"
+    ],
+    "supersededBy": null,
+    "markdown": "\n# CustomResourceDefinition Conversion Webhook\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Definitions](#definitions)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n- [Detailed Design](#detailed-design)\n  - [CRD API Changes](#crd-api-changes)\n  - [Top level fields to Per-Version fields](#top-level-fields-to-per-version-fields)\n  - [Support Level](#support-level)\n  - [Rollback](#rollback)\n  - [Webhook Request/Response](#webhook-requestresponse)\n  - [Metadata](#metadata)\n  - [Monitorability](#monitorability)\n  - [Error Messages](#error-messages)\n  - [Caching](#caching)\n- [Examples](#examples)\n  - [Example of Writing Conversion Webhook](#example-of-writing-conversion-webhook)\n- [Example of Updating CRD from one to two versions](#example-of-updating-crd-from-one-to-two-versions)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Summary\n\n\nThis document proposes a detailed plan for adding support for version-conversion\nof Kubernetes resources defined via Custom Resource Definitions (CRD).  The API\nServer is extended to call out to a webhook at appropriate parts of the handler\nstack for CRDs.\n\nNo new resources are added; the \n[CRD resource](https://github.com/kubernetes/kubernetes/blob/34383aa0a49ab916d74ea897cebc79ce0acfc9dd/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go#L187)\nis extended to include conversion information as well as multiple schema\ndefinitions, one for each apiVersion that is to be served.\n\n## Definitions\n\n**Webhook Resource**: a Kubernetes resource (or portion of a resource) that informs the API Server that it should call out to a Webhook Host for certain operations. \n\n**Webhook Host**: a process / binary which accepts HTTP connections, intended to be called by the Kubernetes API Server as part of a Webhook.\n\n**Webhook**: In Kubernetes, refers to the idea of having the API server make an HTTP request to another service at a point in its request processing stack.  Examples are [Authentication webhooks](https://kubernetes.io/docs/reference/access-authn-authz/webhook/) and [Admission Webhooks](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/).  Usually refers to the system of Webhook Host and Webhook Resource together, but occasionally used to mean just Host or just Resource.\n\n**Conversion Webhook**: Webhook that can convert an object from one version to another.\n\n**Custom Resource**: In the context of this document, it refers to resources defined as Custom Resource Definition (in contrast with extension API server’s resources).\n\n**CRD Package**: CRD definition, plus associated controller deployment, RBAC roles, etc, which is released by a developer who uses CRDs to create new APIs.\n\n## Motivation\n\nVersion conversion is, in our experience, the most requested improvement to CRDs.  Prospective CRD users want to be certain they can evolve their API before they start down the path of developing a CRD + controller. \n\n### Goals\n\n* As an existing author of a CRD, I can update my API's schema, without breaking existing clients.  To that end, I can write a CRD(s) that supports one kind with two (or more) versions.  Users of this API can access an object via either version (v1 or v2), and are accessing the same underlying storage (assuming that I have properly defined how to convert between v1 and v2.)\n\n* As a prospective user of CRDs, I don't know what schema changes I may need in the future, but I want to know that they will be possible before I chose CRDs (over EAS, or over a non-Kubernetes API).\n\n* As an author of a CRD Package, my users can upgrade to a new version of my package, and can downgrade to a prior version of my package (assuming that they follow proper upgrade and downgrade procedures; these should not require direct etcd access.)\n\n* As a user, I should be able to request CR in any supported version defined by CRD and get an object has been properly converted to the requested version (assuming the CRD Package Author has properly defined how to convert).\n\n* As an author of a CRD that does not use validation, I can still have different versions which undergo conversion.\n\n* As a user, when I request an object, and webhook-conversion fails, I get an error message that helps me understand the problem.\n\n* As an API machinery code maintainer, this change should not make the API machinery code harder to maintain\n\n* As a cluster owner, when I upgrade to the version of Kubernetes that supports CRD multiple versions, but I don't use the new feature, my existing CRDs work fine.  I can roll back to the previous version without any special action.\n\n### Non-Goals\n\n## Proposal\n\n1. A CRD object now represents a group/kind with one or more versions.\n\n2. The CRD API (CustomResourceDefinitionSpec) is extended as follows:\n\n    1. It has a place to register 1 webhook.\n\n    2. it holds multiple \"versions\".\n\n    3. Some fields which were part of the .spec are now per-version; namely Schema, Subresources, and AdditionalPrinterColumns.\n\n3. A Webhook Host is used to do conversion for a CRD.\n\n    4. CRD authors will need to write a Webhook Host that accepts any version and returns any version.\n\n    5. Toolkits like kube-builder and operator-sdk are expected to provide flows to assist users to generate Webhook Hosts.\n\n## Detailed Design\n\n### CRD API Changes\n\nThe CustomResourceDefinitionSpec is extended to have a new section where webhooks are defined: \n\n```golang\n// CustomResourceDefinitionSpec describes how a user wants their resource to appear\ntype CustomResourceDefinitionSpec struct {\n  Group string\n  Version string\n  Names CustomResourceDefinitionNames\n  Scope ResourceScope\n  // Optional, can only be provided if per-version schema is not provided.\n  Validation *CustomResourceValidation\n  // Optional, can only be provided if per-version subresource is not provided.\n  Subresources *CustomResourceSubresources\n  Versions []CustomResourceDefinitionVersion\n  // Optional, can only be provided if per-version additionalPrinterColumns is not provided.  \n  AdditionalPrinterColumns []CustomResourceColumnDefinition\n\n  Conversion *CustomResourceConversion\n}\n\ntype CustomResourceDefinitionVersion struct {\n  Name string\n  Served Boolean\n  Storage Boolean\n  // Optional, can only be provided if top level validation is not provided.\n  Schema *JSONSchemaProp\n  // Optional, can only be provided if top level subresource is not provided.\n  Subresources *CustomResourceSubresources\n  // Optional, can only be provided if top level additionalPrinterColumns is not provided.  \n  AdditionalPrinterColumns []CustomResourceColumnDefinition\n}\n\nType CustomResourceConversion struct {\n  // Conversion strategy, either \"nop\" or \"webhook\". If webhook is set, Webhook field is required.\n  Strategy string\n\n  // Additional information for external conversion if strategy is set to external\n  // +optional\n  Webhook *CustomResourceConversionWebhook\n}\n\ntype CustomResourceConversionWebhook {\n  // ClientConfig defines how to communicate with the webhook. This is the same config used for validating/mutating webhooks.\n  ClientConfig WebhookClientConfig\n}\n```\n\n### Top level fields to Per-Version fields\n\nIn *CRD v1beta1* (apiextensions.k8s.io/v1beta1) there are per-version schema, additionalPrinterColumns or subresources (called X in this section) defined and these validation rules will be applied to them:\n\n* Either top level X or per-version X can be set, but not both. This rule applies to individual X’s not the whole set. E.g. top level schema can be set while per-version subresources are set.\n* per-version X cannot be the same. E.g. if all per-version schema are the same, the CRD object will be rejected with an error message asking the user to use the top level schema.\n\nIn *CRD v1* (apiextensions.k8s.io/v1), there will be only version list with no top level X. The second validation guarantees a clean moving to v1. These are conversion rules:\n\n*v1beta1-\u003ev1:*\n\n* If top level X is set in v1beta1, then it will be copied to all versions in v1.\n* If per-version X are set in v1beta1, then they will be used for per-version X in v1.\n\n*v1-\u003ev1beta1:*\n\n* If all per-version X are the same in v1, they will be copied to top level X in v1beta1\n* Otherwise, they will be used as per-version X in v1beta1\n\n### Support Level\n\nThe feature will be alpha in the first implementation and will have a feature gate that is defaulted to false. The roll-back story with a feature gate is much more clear. if we have the features as alpha in kubernetes release Y (\u003eX where the feature is missing) and we make it beta in kubernetes release Z, it is not safe to use the feature and downgrade from Y to X but the feature is alpha in Y which is fine. It is safe to downgrade from Z to Y (given that we enable the feature gate in Y) and that is desirable as the feature is beta in Z.\nOn downgrading from a Z to Y, stored CRDs can have per-version fields set. While the feature gate can be off on Y (alpha cluster), it is dangerous to disable per-version Schema Validation or Status subresources as it makes the status field mutable and validation on CRs will be disabled. Thus the feature gate in Y only protects adding per-version fields not the actual behaviour. Thus if the feature gate is off in Y:\n\n* Per-version X cannot be set on CRD create (per-version fields are auto-cleared).\n* Per-version X can only be set/changed on CRD update *if* the existing CRD object already has per-version X set.\n\nThis way even if we downgrade from Z to Y, per-version validations and subresources will be honored. This will not be the case for webhook conversion itself. The feature gate will also protect the implementation of webhook conversion and alpha cluster with disabled feature gate will return error for CRDs with webhook conversion (that are created with a future version of the cluster).\n\n### Rollback\n\nUsers that need to rollback to version X (but may currently be running version Y \u003e X) of apiserver should not use CRD Webhook Conversion if X is not a version that supports these features.  If a user were to create a CRD that uses CRD Webhook Conversion and then rolls back to version X that does not support conversion then the following would happen:\n\n1. The stored custom resources in etcd will not be deleted.\n\n2. Any clients that try to get the custom resources will get a 500 (internal server error). this is distinguishable from a deleted object for get and the list operation will also fail. That means the CRD is not served at all and Clients that try to garbage collect related resources to missing CRs should be aware of this. \n\n3. Any client (e.g. controller) that tries to list the resource (in preparation for watching it) will get a 500 (this is distinguishable from an empty list or a 404).\n\n4. If the user rolls forward again, then custom resources will be served again.\n\nIf a user does not use the webhook feature but uses the versioned schema, additionalPrinterColumns, and/or subresources and rollback to a version that does not support them per-version, any value set per-version will be ignored and only values in top level spec.* will be honor.\n\nPlease note that any of the fields added in this design that is not supported in previous kubernetes releases can be removed on an update operation (e.g. status update). The kubernetes release where defined the types but gate them with an alpha feature gate, however, can keep these fields but ignore there value.\n\n### Webhook Request/Response\n\nThe Conversion request and response would be similar to [Admission webhooks](https://github.com/kubernetes/kubernetes/blob/951962512b9cfe15b25e9c715a5f33f088854f97/staging/src/k8s.io/api/admission/v1beta1/types.go#L29). The AdmissionReview seems to be redundant but used by other Webhook APIs and added here for consistency.\n\n```golang\n// ConversionReview describes a conversion request/response.\ntype ConversionReview struct {\n  metav1.TypeMeta\n  // Request describes the attributes for the conversion request.\n  // +optional\n  Request *ConversionRequest\n  // Response describes the attributes for the conversion response.\n  // +optional\n  Response *ConversionResponse\n}\n\ntype ConversionRequest struct {\n  // UID is an identifier for the individual request/response. Useful for logging.\n  UID types.UID\n  // The version to convert given object to. E.g. \"stable.example.com/v1\"\n  APIVersion string\n  // Object is the CRD object to be converted.\n  Object runtime.RawExtension\n}\n\ntype ConversionResponse struct {\n  // UID is an identifier for the individual request/response.\n  // This should be copied over from the corresponding ConversionRequest.\n  UID types.UID\n  // ConvertedObject is the converted version of request.Object.\n  ConvertedObject runtime.RawExtension\n}\n```\n\nIf the conversion is failed, the webhook should fail the HTTP request with a proper error code and message that will be used to create a status error for the original API caller.\n\n### Metadata\n\nTo ensure consistent behaviour of converted CRDs following the established API machinery semantics, we will\n\n* check that the returned list of converted objects has the same length and each object (at each index) has the same kind, name, namespace and UID as in the request object list (which ensures the same order), and fail the conversion otherwise\n* restrict which fields of `ObjectMeta` a conversion webhook can change to labels and annotations.\n\n  Mutations to other fields under `metadata` are ignored (i.e. restored to the values of the original object). This means that mutations to other fields also do not lead to an error (we do this because it is hard to define what a change to `ObjectMeta` actually is, with the known encoding issues of empty and undefined lists and maps in mind).\n  \n  Labels and annotations are validated with the usual API machinery ObjectMeta validation semantics.\n\n### Monitorability\n\nThere should be prometheus variables to show:\n\n* CRD conversion latency\n    * Overall\n    * By webhook name\n    * By request (sum of all conversions in a request)\n    * By CRD\n* Conversion Failures count\n    * Overall\n    * By webhook name\n    * By CRD\n* Timeout failures count\n    * Overall\n    * By webhook name\n    * By CRD\n\nAdding a webhook dynamically adds a key to a map-valued prometheus metric. Webhook host process authors should consider how to make their webhook host monitorable: while eventually we hope to offer a set of best practices around this, for the initial release we won’t have requirements here.\n\n\n### Error Messages\n\nWhen a conversion webhook fails, e.g. for the GET operation, then the error message from the apiserver to its client should reflect that conversion failed and include additional information to help debug the problem. The error message and HTTP error code returned by the webhook should be included in the error message API server returns to the user.  For example:\n\n```bash\n$ kubectl get mykind somename\nerror on server: conversion from stored version v1 to requested version v2 for somename: \"408 request timeout\" while calling service \"mywebhookhost.somens.cluster.local:443\"\n```\n\n\nFor operations that need more than one conversion (e.g. LIST), no partial result will be returned. Instead the whole operation will fail the same way with detailed error messages. To help debugging these kind of operations, the UID of the first failing conversion will also be included in the error message. \n\n\n### Caching\n\nNo new caching is planned as part of this work, but the API Server may in the future cache webhook POST responses.\n\nMost API operations are reads.  The most common kind of read is a watch.  All watched objects are cached in memory. For CRDs, the cache\nis per-version. That is the result of having one [REST store object](https://github.com/kubernetes/kubernetes/blob/3cb771a8662ae7d1f79580e0ea9861fd6ab4ecc0/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/etcd.go#L72) per-version which\nwas an arbitrary design choice but would be required for better caching with webhook conversion. In this model, each GVK is cached, regardless of whether some GVKs share storage.  Thus, watches do not cause conversion.  So, conversion webhooks will not add overhead to the watch path.  Watch cache is per api server and eventually consistent.\n\nNon-watch reads are also cached (if requested resourceVersion is 0 which is true for generated informers by default, but not for calls like `kubectl get ...`, namespace cleanup, etc). The cached objects are converted and per-version (TODO: fact check). So, conversion webhooks will not add overhead here too.\n\nIf in the future this proves to be a performance problem, we might need to add caching later.  The Authorization and Authentication webhooks already use a simple scheme with APIserver-side caching and a single TTL for expiration.  This has worked fine, so we can repeat this process.  It does not require Webhook hosts to be aware of the caching.\n\n\n## Examples\n\n\n### Example of Writing Conversion Webhook\n\nData model for v1:\n\n|data model for v1|\n```yaml      \nproperties:\n  spec:\n    properties:\n      cronSpec:\n        type: string\n      image:\n        type: string\n```\n\n|data model for v2|\n```yaml\nproperties:\n  spec:\n    properties:\n      min:\n        type: string\n      hour:\n        type: string\n      dayOfMonth:\n        type: string\n      month:\n        type: string\n      dayOfWeek:\n        type: string\n      image:\n        type: string\n```\n\n\nBoth schemas can hold the same data (assuming the string format for V1 was a valid format).\n\n|crontab_conversion.go|\n\n```golang\nimport .../types/v1\nimport .../types/v2\n\n// Actual conversion methods\n\nfunc convertCronV1toV2(cronV1 *v1.Crontab) (*v2.Crontab, error) {\n  items := strings.Split(cronV1.spec.cronSpec, \" \")\n  if len(items) != 5 {\n     return nil, fmt.Errorf(\"invalid spec string, needs five parts: %s\", cronV1.spec.cronSpec)\n  }\n  return \u0026v2.Crontab{\n     ObjectMeta: cronV1.ObjectMeta,\n     TypeMeta: metav1.TypeMeta{\n        APIVersion: \"stable.example.com/v2\",\n        Kind: cronV1.Kind,\n     },\n     spec: v2.CrontabSpec{\n        image: cronV1.spec.image,\n        min: items[0],\n        hour: items[1],\n        dayOfMonth: items[2],\n        month: items[3],\n        dayOfWeek: items[4],\n     },\n  }, nil\n\n}\n\nfunc convertCronV2toV1(cronV2 *v2.Crontab) (*v1.Crontab, error) {\n  cronspec := cronV2.spec.min + \" \"\n  cronspec += cronV2.spec.hour + \" \"\n  cronspec += cronV2.spec.dayOfMonth + \" \"\n  cronspec += cronV2.spec.month + \" \"\n  cronspec += cronV2.spec.dayOfWeek\n  return \u0026v1.Crontab{\n     ObjectMeta: cronV2.ObjectMeta,\n     TypeMeta: metav1.TypeMeta{\n        APIVersion: \"stable.example.com/v1\",\n        Kind: cronV2.Kind,\n     },\n     spec: v1.CrontabSpec{\n        image: cronV2.spec.image,\n        cronSpec: cronspec,\n     },\n  }, nil\n}\n\n// The rest of the file can go into an auto generated framework\n\nfunc serveCronTabConversion(w http.ResponseWriter, r *http.Request) {\n  request, err := readConversionRequest(r)\n  if err != nil {\n     reportError(w, err)\n  }\n  response := ConversionResponse{}\n  response.UID = request.UID\n  converted, err := convert(request.Object, request.APIVersion)\n  if err != nil {\n     reportError(w, err)\n  }\n  response.ConvertedObject = *converted\n  writeConversionResponse(w, response)\n}\n\nfunc convert(in runtime.RawExtension, version string) (*runtime.RawExtension, error) {\n  inApiVersion, err := extractAPIVersion(in)\n  if err != nil {\n     return nil, err\n  }\n  switch inApiVersion {\n  case \"stable.example.com/v1\":\n     var cronV1 v1Crontab\n     if err := json.Unmarshal(in.Raw, \u0026cronV1); err != nil {\n        return nil, err\n     }\n     switch version {\n     case \"stable.example.com/v1\":\n        // This should not happened as API server will not call the webhook in this case\n        return \u0026in, nil\n     case \"stable.example.com/v2\":\n        cronV2, err := convertCronV1toV2(\u0026cronV1)\n        if err != nil {\n           return nil, err\n        }\n        raw, err := json.Marshal(cronV2)\n        if err != nil {\n           return nil, err\n        }\n        return \u0026runtime.RawExtension{Raw: raw}, nil\n     }\n  case \"stable.example.com/v2\":\n     var cronV2 v2Crontab\n     if err := json.Unmarshal(in.Raw, \u0026cronV2); err != nil {\n        return nil, err\n     }\n     switch version {\n     case \"stable.example.com/v2\":\n        // This should not happened as API server will not call the webhook in this case\n        return \u0026in, nil\n     case \"stable.example.com/v1\":\n        cronV1, err := convertCronV2toV1(\u0026cronV2)\n        if err != nil {\n           return nil, err\n        }\n        raw, err := json.Marshal(cronV1)\n        if err != nil {\n           return nil, err\n        }\n        return \u0026runtime.RawExtension{Raw: raw}, nil\n     }\n  default:\n     return nil, fmt.Errorf(\"invalid conversion fromVersion requested: %s\", inApiVersion)\n  }\n  return nil, fmt.Errorf(\"invalid conversion toVersion requested: %s\", version)\n}\n\nfunc extractAPIVersion(in runtime.RawExtension) (string, error) {\n  object := unstructured.Unstructured{}\n  if err := object.UnmarshalJSON(in.Raw); err != nil {\n     return \"\", err\n  }\n  return object.GetAPIVersion(), nil\n}\n```\n\nNote: not all code is shown for running a web server.  \n\nNote: some of this is boilerplate that we expect tools like Kubebuilder will handle for the user.\n\nAlso some appropriate tests, most importantly round trip test:\n\n|crontab_conversion_test.go|\n|-|\n\n```golang\nfunc TestRoundTripFromV1ToV2(t *testing.T) {\n  testObj := v1.Crontab{\n     ObjectMeta: metav1.ObjectMeta{\n        Name: \"my-new-cron-object\",\n     },\n     TypeMeta: metav1.TypeMeta{\n        APIVersion: \"stable.example.com/v1\",\n        Kind: \"CronTab\",\n     },\n     spec: v1.CrontabSpec{\n        image: \"my-awesome-cron-image\",\n        cronSpec: \"* * * * */5\",\n     },\n  }\n  testRoundTripFromV1(t, testObj)\n}\n\nfunc testRoundTripFromV1(t *testing.T, v1Object v1.CronTab) {\n  v2Object, err := convertCronV1toV2(v1Object)\n  if err != nil {\n     t.Fatalf(\"failed to convert v1 crontab to v2: %v\", err)\n  }\n  v1Object2, err := convertCronV2toV1(v2Object)\n  if err != nil {\n     t.Fatalf(\"failed to convert v2 crontab to v1: %v\", err)\n  }\n  if !reflect.DeepEqual(v1Object, v1Object2) {\n     t.Errorf(\"round tripping failed for v1 crontab. v1Object: %v, v2Object: %v, v1ObjectConverted: %v\",\n        v1Object, v2Object, v1Object2)\n  }\n}\n```\n\n## Example of Updating CRD from one to two versions \n\nThis example uses some files from previous section.\n\n**Step 1**: Start from a CRD with only one version  \n\n|crd1.yaml|\n|-|\n\n```yaml\napiVersion: apiextensions.k8s.io/v1beta1\nkind: CustomResourceDefinition\nmetadata:\n  name: crontabs.stable.example.com\nspec:\n  group: stable.example.com\n  versions:\n  - name: v1\n    served: true\n    storage: true\n    schema:\n      properties:\n        spec:\n          properties:\n            cronSpec:\n              type: string\n            image:\n              type: string\n  scope: Namespaced\n  names:\n    plural: crontabs\n    singular: crontab\n    kind: CronTab\n    shortNames:\n    - ct\n```\n\nAnd create it:\n\n```bash\nKubectl create -f crd1.yaml\n```\n\n(If you have an existing CRD installed prior to the version of Kubernetes that supports the \"versions\" field, then you may need to move version field to a single item in the list of versions or just try to touch the CRD after upgrading to the new Kubernetes version which will result in the versions list being defaulted to a single item equal to the top level spec values)\n\n**Step 2**: Create a CR within that one version:\n\n|cr1.yaml|\n|-|\n```yaml\n\napiVersion: \"stable.example.com/v1\"\nkind: CronTab\nmetadata:\n  name: my-new-cron-object\nspec:\n  cronSpec: \"* * * * */5\"\n  image: my-awesome-cron-image\n```\n\nAnd create it:\n\n```bash\nKubectl create -f cr1.yaml\n```\n\n**Step 3**: Decide to introduce a new version of the API.\n\n**Step 3a**: Write a new OpenAPI data model for the new version (see previous section).  Use of a data model is not required, but it is recommended.\n\n**Step 3b**: Write conversion webhook and deploy it as a service named `crontab_conversion`\n\nSee the \"crontab_conversion.go\" file in the previous section.\n\n**Step 3c**: Update the CRD to add the second version.\n\nDo this by adding a new item to the \"versions\" list, containing the new data model:\n\n|crd2.yaml|\n|-|\n```yaml\n\napiVersion: apiextensions.k8s.io/v1beta1\nkind: CustomResourceDefinition\nmetadata:\n  name: crontabs.stable.example.com\nspec:\n  group: stable.example.com\n  versions:\n  - name: v1\n    served: true\n    storage: false\n    schema:\n      properties:\n        spec:\n          properties:\n            cronSpec:\n              type: string\n            image:\n              type: string\n  - name: v2\n    served: true\n    storage: true\n    schema:\n      properties:\n        spec:\n          properties:\n            min:\n              type: string\n            hour:\n              type: string\n            dayOfMonth:\n              type: string\n            month:\n              type: string\n            dayOfWeek:\n              type: string\n            image:\n              type: string\n  scope: Namespaced\n  names:\n    plural: crontabs\n    singular: crontab\n    kind: CronTab\n    shortNames:\n    - ct\n  conversion:\n    strategy: external\n    webhook:\n      client_config:\n        namespace: crontab\n        service: crontab_conversion\n        Path: /crontab_convert\n```\n\nAnd apply it:\n\n```bash\nKubectl apply -f crd2.yaml\n```\n\n**Step 4**: add a new CR in v2:\n\n|cr2.yaml|\n|-|\n```yaml\n\napiVersion: \"stable.example.com/v2\"\nkind: CronTab\nmetadata:\n  name: my-second-cron-object\nspec:\n  min: \"*\"\n  hour: \"*\"\n  day_of_month: \"*\"\n  dayOfWeek: \"*/5\"\n  month: \"*\"\n  image: my-awesome-cron-image\n```\n\nAnd create it:\n\n```bash\nKubectl create -f cr2.yaml\n```\n\n**Step 5**: storage now has two custom resources in two different versions. To downgrade to previous CRD, one can apply crd1.yaml but that will fail as the status.storedVersions has both v1 and v2 and those cannot be removed from the spec.versions list. To downgrade, first create a crd2-b.yaml file that sets v1 as storage version and apply it, then follow \"*Upgrade existing objects to a new stored version*\" in [this document](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/). After all CRs in the storage has v1 version, you can apply crd1.yaml.\n\n**Step 5 alternative**: create a crd1-b.yaml that has v2 but not served.\n\n|crd1-b.yaml|\n|-|\n```yaml\n\napiVersion: apiextensions.k8s.io/v1beta1\nkind: CustomResourceDefinition\nmetadata:\n  name: crontabs.stable.example.com\nspec:\n  group: stable.example.com\n  versions:\n  - name: v1\n    served: true\n    storage: true\n    schema:\n      properties:\n        spec:\n          properties:\n            cronSpec:\n              type: string\n            image:\n              type: string\n  - name: v2\n    served: false\n    storage: false\n  scope: Namespaced\n  names:\n    plural: crontabs\n    singular: crontab\n    kind: CronTab\n    shortNames:\n    - ct\n  conversion:\n    strategy: external\n    webhook:\n      client_config:\n        namespace: crontab\n        service: crontab_conversion\n        Path: /crontab_convert\n```\n\n### Risks and Mitigations\n\nWhat are the risks of this proposal and how do we mitigate.\nThink broadly.\nFor example, consider both security and how this will impact the larger kubernetes ecosystem.\n\nHow will security be reviewed and by whom?\nHow will UX be reviewed and by whom?\n\nConsider including folks that also work outside the SIG or subproject.\n\n## Graduation Criteria\n\nv1beta1:\n\n- Finalize validation restrictions on what metadata fields conversion is able to mutate (https://github.com/kubernetes/kubernetes/issues/72160).\n- Ensure the scenarios from (https://github.com/kubernetes/kubernetes/issues/64136) are tested:\n  - Ensure what is persisted in etcd matches the storage version\n  - Set up a CRD, persist some data, change the storage version, and ensure the previously persisted data is readable\n  - Ensure discovery docs track a CRD through creation, version addition, version removal, and deletion\n  - Ensure `spec.served` is respected\n- Error-case handling when calling conversion results in an error during \n  - {get, list, create, update, patch, delete, deletecollection} calls to the primary resource\n  - {get, update, patch} calls to the status subresource\n  - {get, update, patch} calls to the scale subresource\n\nv1:\n\n- ConversionReview API v1 is stable (including fields that need to be updated for v1beta1 or v1)\n- Documented step-by-step CRD version migration workflow that covers the entire\n  process of migrating from one version to another (introduce a new CRD version,\n  add the converter, migrate clients, migrate to new storage version, all the\n  way to removing the old version)\n\n## Implementation History\n\n- Implemented in Kubernetes 1.13 release (https://github.com/kubernetes/kubernetes/pull/67006)\n\n## Alternatives\n\nFirst a defaulting approach is considered which per-version fields would be defaulted to top level fields. but that breaks backward incompatible change; Quoting from API [guidelines](/contributors/devel/sig-architecture/api_changes.md#backward-compatibility-gotchas):\n\n\u003e A single feature/property cannot be represented using multiple spec fields in the same API version simultaneously\n\nHence the defaulting either implicit or explicit has the potential to break backward compatibility as we have two sets of fields representing the same feature.\n\nThere are other solution considered that does not involved defaulting:\n\n* Field Discriminator: Use `Spec.Conversion.Strategy` as discriminator to decide which set of fields to use. This approach would work but the proposed solution is keeping the mutual excusivity in a broader sense and is preferred.\n* Per-version override: If a per-version X is specified, use it otherwise use the top level X if provided. While with careful validation and feature gating, this solution is also backward compatible, the overriding behaviour need to be kept in CRD v1 and that looks too complicated and not clean to keep for a v1 API.\n\nRefer to [this document](http://bit.ly/k8s-crd-per-version-defaulting) for more details and discussions on those solutions.\n"
+  },
+  {
+    "id": "0d47d091df541769999f19b8d91c45ee",
+    "title": "Vanilla CRD OpenAPI Subset: Structural Schemas",
+    "authors": ["@sttts", "@mbohlool"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["sig-api-machinery"],
+    "reviewers": ["@liggitt", "@apelisse", "@mbohlool", "@DirectXMan12"],
+    "approvers": ["@lavalamp", "@deads2k"],
+    "editor": "TBD",
+    "creationDate": "2019-04-25",
+    "lastUpdated": "2019-05-09",
+    "status": "implemented",
+    "seeAlso": [
+      "/keps/sig-api-machinery/20180415-crds-to-ga.md",
+      "https://github.com/kubernetes/enhancements/pull/926",
+      "https://github.com/kubernetes/enhancements/pull/709"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Vanilla CRD OpenAPI subset: structural schemas\n\nOpenAPI has the goal of describing every API possible. CRDs have the goal of presenting a consistent API to users of Kubernetes. This KEP proposes a restriction of permissible CRD OpenAPI validation schemas to match the later and to enable us add advanced CRD features like server-side pruning, defaulting, apply and potentially Protobuf support in the future.\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n  - [Context](#context)\n    - [Consumers of the restricted OpenAPI Schema](#consumers-of-the-restricted-openapi-schema)\n    - [Producers of the restricted OpenAPI Schema](#producers-of-the-restricted-openapi-schema)\n  - [Criteria](#criteria)\n- [Proposal](#proposal)\n  - [Formal Definition](#formal-definition)\n  - [Metadata](#metadata)\n  - [Unfolding the Extensions](#unfolding-the-extensions)\n  - [Litmus Test Examples](#litmus-test-examples)\n- [Design Details](#design-details)\n  - [Scope of necessary Changes](#scope-of-necessary-changes)\n  - [API Compatibility Plan](#api-compatibility-plan)\n  - [Implementation Plan](#implementation-plan)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThe CRD validation schemas today support nearly the whole OpenAPI v3 schema language. Here, we propose to impose a restriction called _structural schema_ to that language, which will \n* significantly reduce difficulty in moving forward with advanced features like pruning, defaulting, server-side apply, read-only fields and protobuf in the near and medium term future\n* while keeping needed expressivity to express Kubernetes-like APIs including powerful value validations.\n\nSchema generators (like [crd-gen](https://github.com/kubernetes-sigs/controller-tools/tree/master/pkg/crd) and [openapi-gen](https://github.com/kubernetes/kube-openapi/tree/master/cmd/openapi-gen)) which recurse over API Golang types produce structural schemas naturally. Developers can enrich those structural schemas with nearly arbitrary value validation logic, usually using in-code `+kubebuilder`-like tags.\n\nThe restriction is an API change of the CustomResourceDefinition kind in the `apiextensions.k8s.io/v1beta1` API group, but in the sense that\n* CRDs created via the `v1beta1` endpoints are not restricted such that existing CRDs keep working **as before**,\n* but CRDs created or updated via the `v1beta1` endpoints have to conform to the new restrictions **if they want to make use of new features**.\n\nIn the `v1` CustomResourceDefinition API we will enforce structural schemas on creation and do ratcheting validation on update (allow updates with non-structural schemas if the original schema was non-structural already).\n\nTo communicate to the developer that a CRD does not follow the structural schema restriction and will therefore not benefit from future developments, we add a `NonStructuralSchema` condition to the CRD.\n\n## Motivation\n\n* OpenAPI has the goal of describing every API possible. We have the goal of presenting a consistent API to users of Kubernetes. OpenAPI therefore makes a fantastic output format able to describe anything, but a poor input format, because there's not \"one correct way\" to do a given thing. It is easy to accidentally (or intentionally) describe things in OpenAPI v3 that are not similar to other Kubernetes APIs and far outside of the [Kuberntes API conversions](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md).\n* OpenAPI v3 is a very open and wide spec that can define crazy things using logical operators (`not(anyOf(oneof ...))`), leading to complicated semantics, making advanced CRD features difficult or even impossible to implement.\n\n### Goals\n\n* Restrict CRD validation OpenAPI v3 schemas to give one canonical way to express objects we intend to support as Kubernetes-like types.\n* Simplify the semantics of CRD validation OpenAPI v3 schemas to make implementation of advanced feature like pruning, defaulting, server-side apply and read-only validation considerably less difficult.\n* Support embedding of existing API types into a CRD via RawExtensions or via Golang type reuse (e.g. of PodSpecs).\n* Allow a rich OpenAPI v3 validation schema for CustomResource validation, but allow us to programmatically derive the restricted spec for all other purposes.\n* Prospective goal: have a canonical mapping from a restricted schema to Golang types and Protobuf v3.\n\n### Non-Goals\n\n* a human readable spec and convenience to write. We focus on machine readability and clear and simple semantics.\n* model all kind of crazy JSON objects. We focus on Kubernetes-like objects with an OpenAPI schema derived via code generation (crd-gen) from Golang types.\n\n### Context\n\nWe assume the following context of OpenAPI schemas in the Kubernetes architecture and ecosystem:\n\n#### Consumers of the restricted OpenAPI Schema\n\n* server side validation (consumes structural schema including value validation)\n* OpenAPI publishing\n  * client side validation\n  * kubectl explain\n  * client generators (https://github.com/kubernetes-client)\n  * doc generator\n  * kube-aggregator\n  * IDEs like IntelliJ for input completion\n* server side apply\n* pruning\n* defaulting\n* possibly in the future:\n  * server side read-only update validation\n  * Protobuf publishing\n\n#### Producers of the restricted OpenAPI Schema\n\n* kube-openapi's **openapi-gen** mechanically producing go-openapi definitions for native types compiled into kube-apiserver\n* **crd-gen/crd-schema-gen** in https://github.com/kubernetes-sigs/controller-tools mechanically producing CRD validation schemas from Golang types with tags\n* a **developer** adding custom OpenAPI schemas for openapi-gen or crd-gen/crd-schema-gen.\n* a **developer** manually writing complete CRD validation schemas.\n\n![Information flow of OpenAPI schemas in Kubernetes](20190425-structural-openapi-context.png)\n\n### Criteria\n\nTo allow only a subset of features of OpenAPI v3, we need to have guidelines for what features we want to restrict and what features we need to permit. We would like to select a single canonical way in OpenAPI to represent these Kubernetes-like types:\n\n* non-polymorphics types: primitive types, objects, strings maps, arrays.\n\n  Examples:\n  * string maps like labels and annotations, but also `map[string]int64`\n  * IPv4 vs IPv6 strings: same fields with different formats\n  * embedded PodSpec (a non-trivial object)\n  \n* discriminated union types (as described in [KEP: Union Types](https://github.com/kubernetes/enhancements/pull/926)).\n  \n  Examples:\n  * VolumeSource, DeploymentStrategy, …\n\n* a very restrictive, fixed set of polymorphic types.\n \n  Examples:\n  * IntOrString\n  * RawExtensions, i.e. unconstrained, embedded toplevel types\n  * arbitrary JSON\n  \nNote: we explicitly accept that polymorphic types (like IntOrString) are an anti-pattern for Kubernetes-like types and we do not want to encourage its use beyond IntOrString.\n\nAs a soft criterion to validate the direction, restricted CRD OpenAPI schemas should represent a model that can also be represented with other parts of the system: \n\n    Golang types \u003c=\u003e OpenAPI \u003c=\u003e Protobuf.\n\nEach of these is to be restricted to lead to a canonical mapping.\n\n## Proposal\n\nWe propose to define a subset of OpenAPI v3 called _structural schema_ which\n\n* allows to algorithmically derive specified types and object field names\n* without the need of semantical understanding of **all** OpenAPI v3 constructs.\n\nHere is an example of what we want to avoid:\n\n```yaml\nproperties:\n  foo:\n    type: string\n  bar:\n    minimum: 42\nanyOf:\n- properties:\n    foo:\n      pattern: abc\n    bar:\n      type: string\n- properties:\n    bar:\n      type: integer\n      minimum: 1\n      maximum: 0\n```\nIn this example, `.foo` is specified as string even without looking at the `anyOf` construct, but `.bar`'s type is unclear without deeper semantic knowledge of `anyOf` and of minimum and maximum.\n\nWe want to be able to mechanically extract specified object field names and types using only the information specified in `properties`, `items`, `additionalProperties` and `type`. Other constructs like logical junctors should be usable to express complex validations, but should not be essential for the specification of the type structure of JSON values.\n\nWe reach this goal in defining _structural schema_ by following this guiding principle:\n\n**Structural Completeness**: all types and the possible object fields within a JSON object must be apparent from a structural schema without understanding the logical junctors `anyOf`, `allOf`, `oneOf` and `not`.\n\n### Formal Definition\n\nWe define a _structural schema_ by choosing another data structure representation of [`JSONSchemaProps`](https://github.com/kubernetes/kubernetes/blob/d0c3b708026f681f7467d36dcf9696f89ccfcfab/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types_jsonschema.go#L20), while adding vendor extension fields for:\n* `x-kubernetes-preserve-unknown-fields: true` – recursive version of `additionalProperties: true`, influencing pruning to not prune unknown fields (compare [pruning KEP](https://github.com/kubernetes/enhancements/pull/709) for more details).\n* `x-kubernetes-embedded-resource: true` – marks a RawExtension field to embed a complete `runtime.Object` like object with `TypeMeta` and `ObjectMeta`.\n* `x-kubernetes-unions: [...]` as defined in [KEP: Union Types](https://github.com/kubernetes/enhancements/pull/926)\n* `x-kubernetes-int-or-string: true` – marks a `IntOrString` polymorphic type.\n\nObserve that any validated `JSONSchemaProps` can be represented as `StructuralSchema`. Validation allows us to drop some unsupported fields (which are part of `JSONSchemaProps` but which we forbid in CRDs):\n\n```go\ntype StructuralSchema struct {\n\tItems                *StructuralSchema\n\tProperties           map[string]StructuralSchema\n\n\tGeneric\n\tExtensions\n\t\n\t*ValueValidation\n}\n\ntype Extensions struct {\n\t// x-kubernetes-unions lists unions, i.e. sets of fields which are \n\t// mutual exclusive.\n\tXUnions []Union\n\t\n\t// x-kubernetes-preserve-unknown-fields stops the API server\n\t// decoding step from pruning fields which are not specified\n\t// in the validation schema. This affects fields recursively,\n\t// but switches back to normal pruning behaviour if nested\n\t// properties or additionalProperties are specified in the schema.\n\t// This can either be true or null. False is forbidden.\n        XPreserveUnknownFields *bool\n        \n\t// x-kubernetes-embedded-resource defines that the value is an\n\t// embedded Kubernetes runtime.Object, with TypeMeta and\n\t// ObjectMeta. The type must be object. It is allowed to further\n\t// restrict the embedded object. kind, apiVersion and metadata\n\t// are validated automatically. x-kubernetes-preserve-unknown-fields\n\t// is allowed to be true, but does not have to be if the object\n\t// is fully specified (up to kind, apiVersion, metadata).\n        XEmbeddedResource bool\n        \n\t// x-kubernetes-int-or-string specifies that this value is\n\t// either an integer or a string. If this is true, an empty\n\t// type is allowed and type as child of anyOf is permitted\n\t// if following one of the following patterns:\n\t//\n\t// 1) anyOf:\n\t//    - type: integer\n\t//    - type: string\n\t// 2) allOf:\n\t//    - anyOf:\n\t//      - type: integer\n\t//      - type: string\n\t//    - ... zero or more\n        XIntOrString bool\n}\n\ntype Union struct {\n\t// discriminator is the name of the discriminator field.\n\tDiscriminator string\n\t// fields lists the union field names with their discriminator value.\n\t// This must be undefined if discriminator is empty.\n\tFields map[string]string\n}\n\ntype Generic struct {\n\t// type specifies the type of a value.\n\t// It can be object, array, number, integer, boolean, string.\n\t// It is optional only if x-kubernetes-preserve-unknown-fields \n\t// or x-kubernetes-int-or-string is true.\n\tType        string\n\tAdditionalProperties *StructuralOrBool\n\t\n\tDescription string\n\tTitle       string\n        Nullable    bool\n\tDefault     *JSON\n        ReadOnly    bool\n}\n\ntype ValueValidation struct {\n\tFormat               string\n\tMaximum              *float64\n        ExclusiveMaximum     bool\n        Minimum              *float64\n        ExclusiveMinimum     bool\n        MaxLength            *int64\n        MinLength            *int64\n        Pattern              string\n        MaxItems             *int64\n        MinItems             *int64\n        UniqueItems          bool\n        MultipleOf           *float64\n        Enum                 []JSON\n        MaxProperties        *int64\n        MinProperties        *int64\n        Required             []string\n        AllOf                []NestedValueValidation\n        OneOf                []NestedValueValidation\n        AnyOf                []NestedValueValidation\n        Not                  *NestedValueValidation\n}\n\ntype NestedValueValidation struct {\n\tValueValidation\n\n\tItems      *NestedValueValidation\n        Properties map[string]NestedValueValidation\n        \n        // Anything set in the following will make the scheme \n        // non-structural, with the exception of these two patterns if \n        // x-kubernetes-int-or-string is true:\n        //\n        // 1) anyOf:\n        //    - type: integer\n        //    - type: string\n        // 2) allOf:\n        //    - anyOf:\n        //      - type: integer\n        //      - type: string\n        //    - \u003cNestedValueValidation\u003e\n        ForbiddenGenerics Generic\n        ForbiddenExtensions Extensions\n}\n```\n\nNote: we do not have to change the API types to this schema right away. Take it as hypothetical data structure for the sake of definition, potentially used though during validation.\n\nWe add the following additional validations for CRDs based on the representation as `StructuralSchema`:\n \n1. **structurality:** both `ForbiddenGenerics` and `ForbiddenExtensions` only have zero values, with the two exceptions of (4).\n2. **completeness:** the schema is _structurally complete_\n3. **RawExtension:** for every schema with `x-kubernetes-embedded-resource: true`, `type: object` is set, and either properties are specified or `x-kubernetes-preserve-unknown-fields: true` is set.\n4. **IntOrString:** for `x-kubernetes-int-or-string: true` either `type` is empty under `anyOf` and `allOf` or the schema structure is either\n \n   ```yaml\n   x-kubernetes-int-or-string: true\n   anyOf:\n   - type: integer\n   - type: string\n   ...\n   ```\n   or\n   ```yaml\n   x-kubernetes-int-or-string: true\n   allOf:\n   - anyOf:\n     - type: integer\n     - type: string\n   - ...\n   ...\n   ```\n\nStructurally complete means that any specified JSON path in a schema has a specified type outside of any logical junctors (`andOf`,`anyOf`,`oneOf`,`not`; compare the guiding principle above and its example). In other words, we validate that \n\n- every object field (`properties`), \n- every array value (`items`) and \n- every string map value (`additionalProperties`)\n\nhas a non-empty `type` outside of logical junctors, with the possible exceptions of\n- `x-kubernetes-int-or-string: true` is set in the empty-type schema\n- `x-kubernetes-preserve-unknown-fields: true` is set in the empty-type schema.\n\nIf such a `type` is empty, we report the following structural schema validation error (via the `NonStructuralSchema` condition messages, and via real API rejection according to the [API Compatibility Plan](#api-compatibility-plan)), e.g.:\n\n```\n.properties[foo].items.properties[bar].type must be non-empty\n```\n\nThe additional validations have the following consequences:\n\n1. `additionalProperties`, `description`, `title`, `nullable`, `default`, `readOnly` and `type` (with the two `x-kubernetes-int-or-string: true` exceptions) and the new `x-kubernetes-*` extensions cannot be used inside of logical junctors `allOf`, `anyOf`, `oneOf`, `not`\n2. When setting `StructuralSchema.ValueValidation` recursively to nil in a schema, the schema becomes weaker.\n\nThe second consequence means that we can correctly validate types and the existence of fields without understanding value validations (i.e. without rejecting an object as invalid which is actually valid). This is crucial for knowingly incomplete schema algorithms as used in pruning, defaulting, apply and elsewhere.\n\n### Metadata\n\nThe `metadata` field at the object root is implicitly specified and validated. In order to enforce that every CustomResource is a good citizen in the API Machinery of the API server, i.e. that features like owner references, finalizers, server-side apply etc. work as designed, we do not want that CRDs restrict `metadata` fields other than `name` and `generateName`.\n\nHence, we restrict\n\n1. that `properties[metadata]` at the root of the schema does not specify anything else than `type:object` and the properties `name` and `generateName`, i.e. at most:\n   ```yaml\n   properties:\n     metadata:\n       type: object\n       properties:\n         name: \u003cany-schema\u003e\n         generateName: \u003cany-schema\u003e\n         # nothing else here\n       # nothing else here\n     ...\n   ...\n   ```\n2. that `metadata` at the object root is not specified inside of `ValueValidation`, e.g.\n   ```yaml\n   anyOf:\n     properties:\n       metadata: \u003cany-schema\u003e\n   ...\n   ```\n   is forbidden, but\n   ```yaml\n   anyOf:\n     properties:\n       embedded:\n         x-kubernetes-embedded-resource: true\n         properties:\n           metadata: \u003cany-schema\u003e\n   ...\n   ```\n   is allowed.\n   \nThe restrictions do not apply to embedded resources (specified via `x-kubernetes-embedded-resource`) as these are not instances yet which interact with API machinery beyond pure validation.\n\nNote, that it is discouraged to completely specify `metadata` for embedded resources, but only those restrictions which go beyond the implicit `metadata` validation induced by `x-kubernetes-embedded-resource`.\n\n### Unfolding the Extensions\n\nDuring publishing we unfold the new `x-kubernetes-*` extensions:\n\n* `x-kubernetes-int-or-string: true` unfolds to\n  ```yaml\n  x-kubernetes-int-or-string: true\n  anyOf:\n  - type: integer\n  - type: string\n  ```\n  if `ValueValidation.AnyOf` is unset, otherwise to: \n  ```yaml\n  x-kubernetes-int-or-string: true\n  allOf:\n  - anyOf:\n    - type: integer\n    - type: string\n  - \u003cValueValidation.AnyOf value\u003e\n  ```\n* `x-kubernetes-embedded-resource: true` unfolds to\n  ```yaml\n  x-kubernetes-embedded-resource: true\n  properties:\n    apiVersion:\n      type: string\n      pattern: \u003csome-apiVersion-regex\u003e\n    kind:\n      type: string\n    metadata:\n      type: object\n      \u003csome-further-ObjectMeta-schema\u003e\n  required:\n  - kind\n  - apiVersion\n  ```\n  if the three properties are not already specified, or an analog `allOf` variant logically and'ing the existing and the unfolded schema.\n* `x-kubernetes-unions: [{\"discriminator\":\"type\",\"fields\":{\"foo\":\"Foo\", \"bar\":\"Bar\"}}]` unfolds to\n  ```yaml\n  x-kubernetes-unions:\n  - discriminator: type\n    fields:\n      foo: Foo\n      bar: Bar\n  oneOf:\n    - required: [\"foo\"]\n    - required: [\"bar\"]\n  properties:\n    type:\n      type: string\n      enum:\n      - Foo\n      - Bar\n  ```\n  if no `oneOf` is specified, the analog and'ed variant with `allOf` otherwise.\n\nThese unfoldings are subject to change and explicitly not part of the API, but merely illustration and a rough sketch here. Code generators and other tools must not depend on the unfolding to happen in one way or another. It is guaranteed though that the `x-kubernetes-*` fields are preserved during publishing and hence can be used by tooling as an abstract hint for semantics which are complex to express in OpenAPI. \n\nThe guiding principle is that we don't want OpenAPI-consuming tooling, e.g. client generators, to implement OpenAPI schema pattern heuristics to recover the original types like unions, RawExtensions or `IntOrString`. We support them with our abstract, officially supported vendor extensions for those cases.\n\n### Litmus Test Examples\n\nThe structural schema with these extra validations is expressive enough for the litmus test of types given in the criteria section.\n\nNote: the kubebuilder tags are partially just sketches because the tags in crd-gen are subject to change and are heavily extended right now.\n\n1. discriminated union:\n   ```go\n   type DiscriminatedVolumeSource struct {\n     // +discriminator\n     Type string\n     // +union\n     EmptyDir *EmptyDir\n     // +union\n     HostPath *HostPath\n     // +required\n     SharedField string\n   }\n   \n   type HostPath struct {\n     // +kubebuilder:pattern:\"/.*\"\n     // +required\n     Path string\n   }\n   ```\n   is specified by\n   ```yaml\n   type: object\n   x-kubernetes-unions:\n   - discriminator: type\n     fields:\n       emptyDir: EmptyDir\n       hostPath: HostPath\n   properties:\n     type:\n       type: string\n       enum: [\"EmptyDir\",\"HostPath\"]\n     emptyDir:\n       type: object\n     hostPath:\n       type: object\n       properties:\n         path:\n           type: string\n           pattern: \"/.*\"\n         required: [\"path\"]\n     sharedField:\n       type: string\n   required:\n   - sharedField\n   ```\n   and published as OpenAPI v3 with the following additional schema fields (in OpenAPI v2 this cannot be expressed):\n   ```yaml\n   oneOf:\n   - required: [\"emptyDir\"]\n   - required: [\"hostPath\"]\n   ```\n   After dropping the value validation, we get:\n   ```yaml\n   type: object\n   x-kubernetes-unions:\n   - discriminator: type\n     fields:\n       emptyDir: EmptyDir\n       hostPath: HostPath\n   properties:\n     type:\n       type: string\n     emptyDir:\n       type: object\n     hostPath:\n       type: object\n       properties:\n         path:\n           type: string\n     sharedField:\n       type: string\n   ```\n2. non-polymorphic type, polymorphic format for IPv4 and IPv6:\n   ```go\n   // +kubebuilder:anyOf:pattern=\"^(?:(?:25[0...$\",pattern=\"^(([0-9a-fA-F]...$\"\n   type IPv4Orv6 string\n   ```\n   is specified by:\n   ```yaml\n   type: string\n   anyOf:\n   - pattern: \"^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}...$\"\n   - pattern: \"(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|...$\"\n   ```\n   and published the same way as OpenAPI v3 and as `type: string` in OpenAPI v2 (neither `anyOf` nor `pattern` exist in OpenAPI v2).\n  \n   After dropping the value validation, we get:\n   ```yaml\n   type: string\n   ```\n\n3. RawExtensions, i.e. unconstrained, embedded toplevel types \n   ```go\n   type Foo struct {\n     // +nullable\n     // +kubebuilder:properties=apiVersion:pattern=v1\n     X runtime.RawExtension\n   }\n   ```\n   specified by:\n   ```yaml\n   type: object\n   nullable: true\n   x-kubernetes-embedded-resource: true\n   x-kubernetes-preserve-unknown-fields: true\n   properties:\n     apiVersion:\n       pattern: v1\n   ```\n   and published as OpenAPI v3 with the following additional schema fields:\n   ```yaml\n   properties:\n     apiVersion:\n       allOf:\n       - type: string\n         pattern: \u003csome-apiVersion-regex\u003e\n       - pattern: v1\n     kind:\n       type: string\n     metadata:\n       type: object\n       \u003csome-further-ObjectMeta-schema\u003e\n   ```\n   In OpenAPI v2 `pattern` does not exist and is dropped.\n   \n   After dropping the value validation, we get:\n   ```yaml\n   type: object\n   nullable: true\n   x-kubernetes-embedded-resource: true\n   x-kubernetes-preserve-unknown-fields: true\n   properties:\n     apiVersion:\n       type: string\n     kind:\n       type: string\n     metadata:\n       type: object\n       \u003csome-further-non-value-validation-ObjectMeta-schema\u003e\n   ```\n\n4. embedded JSON, further specified by the user using tags:\n   ```go\n   type Foo struct {\n     // +nullable\n     // +kubebuilder:type=object\n     Json JSON\n   }\n   ```\n   specified by:\n   ```yaml\n   type: object\n   properties:\n     json:\n       type: object\n       x-kubernetes-preserve-unknown-fields: true\n       nullable: true\n   ```\n   and published the same way as OpenAPI v3. For OpenAPI v2 we have to drop `type: object` and `nullable: true` as the latter does not exist in v2 and `type: object` is too restrictive (does not allow `null` as value).\n    \n   Dropping value validation does not change anything.\n\n5. string maps:\n   ```go\n   // +kubebuilder:additionalProperties:minimum=42\n   map[string]int\n   ```\n   specified by:\n   ```yaml\n   type: object\n   additionalProperties:\n     type: integer\n     minimum: 42\n   ```\n   and published as OpenAPI v3 and v2 the same way.\n   \n   After dropping the value validation, we get:\n   ```yaml\n   type: object\n   additionalProperties:\n     type: integer\n   ```\n6. IntOrString:\n   ```go\n   type Foo struct {\n     // +kubebuilder:pattern=abc\n     // +kubebuilder:anyOf=minimum=42,maximum=50;minimum=52,maximum=60\n     X *IntOrString\n   }\n   \n   type IntOrString struct {\n     Integer *int\n     String *string\n   } + marshallers\n   ```\n   specified by:\n   ```yaml\n   x-kubernetes-int-or-string: true\n   pattern: abc\n   anyOf:\n   - minimum: 42\n     maximum: 50\n   - minimum: 52\n     maximum: 60\n   ```\n   or alternatively by:\n   ```yaml\n   x-kubernetes-int-or-string: true\n   allOf:\n   - anyOf: // optionally provided and accepted\n     - type: integer\n     - type: string\n   - pattern: abc\n     anyOf:\n     - minimum: 42\n       maximum: 50\n     - minimum: 52\n       maximum: 60\n   ```\n   The latter is what is also published as OpenAPI v3, but as `x-kubernetes-int-or-string: true` in OpenAPI v2 (neither `anyOf`, nor `pattern` exists in v2).\n   \n   After dropping the value validation, we get:\n   ```yaml\n   x-kubernetes-int-or-string: true\n   ```   \n\nNote: the OpenAPI v2 output given above is the same we get from the [OpenAPI v2 filtering in Kubernetes 1.14](https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/v2/conversion.go#L30): that filtering in the CRD OpenAPI v2 publishing pipeline supports the unrestricted range of the CRD OpenAPI v3 language and turns it into a weaker variant understandable by kubectl 1.13 and 1.14. Hence, our changes above have no influence on the OpenAPI v2 publishing other than passing through of our new `x-kubernetes-*` vendor extensions.\n\nAlso note that structural schemas enforce types and properties outside of logical junctors. OpenAPI v2 publishing will preserve these and hence, structural schemas lead to a more complete OpenAPI v2 spec, client-side validation and `kubectl explain` output.\n\n## Design Details\n\n### Scope of necessary Changes\n\n* To implement this KEP we won't have to modify CustomResource validation in any way. We can keep using the go-openapi based validation algorithm.\n* To implement this KEP we won't have to modify kubectl in any way. We can though add support for the new `x-kubernetes-*` fields, though we will probably wait for OpenAPI v3 publishing and support in kubectl before doing that.\n* We don't have to change the CRD API types as described above in the proposal (only add the vendor extensions), but we can implement the structural schema check via a number of additional CRD validations on-top of the existing CRD API types.\n\n### API Compatibility Plan\n\n* When creating `v1` CRDs, only structural schemas (with value-validation) are allowed.\n* When updating `v1` CRDs, non-structural schemas are only allowed if the existing persisted object contained a non-structural schema already.\n* Any new feature and any feature not promoted to beta yet, will be available for structural schemas only:\n  * webhook conversion:\n    * rejected at CRD validation time on create and on update which sets the webhook; user gets validation error\n    * pre-existing webhooks on update will stay allowed\n  * publishing\n    * not published at publish time; the `NonStructuralSchema` condition is added to CRD status with a warning\n  * pruning (which will be with explicit opt-in in `v1beta1`)\n    * rejected at CRD validation time if pruning is enabled in the CRD; user gets validation error\n  * defaulting\n    * rejected at CRD validation time if defaults are set in the CRD; user gets validation error\n  * `x-kubernetes-unions` field:\n    * rejected at CRD validation time if this field is set; user gets validation error\n\n  This should create natural motivation for the community to phase out old, non-structural schemas without breaking beta compatibility guarantees.\n  \nWe add a condition `NonStructuralSchema` to the CRD objects to reflect that information to the user about a non-structural schema. This condition is only created in case its value is `True`. It is removed if it turns to `False` due to schema updates.\n \n### Implementation Plan\n\n**For Kubernetes 1.15, must haves:**\n\n* Implement structural-schema API validation for 1.15, but don't reject non-structural objects, only set the NonStructuralSchema condition.\n* Do not publish a CRD schema if NonStructuralSchema condition is true (pre-requisite of promoting publishing to beta).\n\n**For Kubernetes 1.15, stretch goals:**\n\n* Do not accept webhook conversion to be set if the schemas are not structural.\n\n  Note: we intend to bring webhook conversion to beta in 1.15. Hence, we won't block webhook conversion on this item, but indent to make it conditional on structural schemas if we are confident enough about the structural schema validation.\n\n**Totally uncommitted for 1.15, but possible as alpha:**\n\n* pruning\n* defaulting\n\n### Test Plan\n\nNext to unit tests, we will add integration tests with real CustomResourceDefinitions. Those will:\n\n- create and update CRDs with \n  * structural\n  * non-structural and\n  * no schema, \n  with \n  * one version and\n  * with multiple versions and different schema\n  and check that the `NonStructuralSchema` condition is set correctly.\n- add integration and/or e2e test-cases to CRD OpenAPI publishing which make sure that publishing is not done for structural schemas. Add tests using the new `x-kubernetes-*` vendor extension fields.\n- add integration and/or e2e test-cases to CRD webhook conversion which make sure that webhooks are rejected on create and on update if a non-structural schema is set.\n\n### Graduation Criteria\n\nThe structural schema feature will graduate with the features which have it as a pre-requisite:\n\n* webhook conversion to beta will depend on a schema being structural. The feature itself does not depend on structural schemas. So promoting webhook conversion to beta with structural schema validation in place has low risk.\n* defaulting/pruning heavily depend on structural schemas. They will be the main test bed of the structural schema code.\n\n### Upgrade / Downgrade Strategy\n\nOn the API surface we add\n* a `NonStructuralSchema` condition. This is binary compatible during upgrade and downgrade, i.e. no special migration steps are necessary for the cluster admin.\n* a `x-kubernetes-*` vendor extensions to the CRD type. These will be dropped on write on downgrade, which is to be expected and accepted for new fields in beta resources.\n\n### Version Skew Strategy\n\n* kubectl 1.14 has to cope with published schemas created from unfolding of the `x-kubernetes-*` fields. The result of unfolding is a valid schema of today's CRD schema language and therefore supported by the OpenAPI v2 publishing algorithm. Hence, unfolding does not add anything new.\n* the new condition is ignored by kubectl 1.14. \n\n## Implementation History\n\n- PR to implement this KEP: https://github.com/kubernetes/kubernetes/pull/77207\n- brain storming document with more detail and background: https://docs.google.com/document/d/1pcGlbmw-2Y0JJs9hsYnSBXamgG9TfWtHY6eh80zSTd8/edit?ts=5cbe0089#heading=h.l1xazlj9aw9f\n- prototype of pruning: https://github.com/kubernetes/kubernetes/pull/64558\n- prototype of defaulting: https://github.com/kubernetes/kubernetes/pull/63604\n"
+  },
+  {
+    "id": "0309dd37068d5073bacb225fbfac2a18",
+    "title": "Defaulting for Custom Resources",
+    "authors": ["@sttts"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["sig-api-machinery"],
+    "reviewers": ["@deads2k", "@lavalamp", "@liggitt", "@mbohlool", "@apelisse"],
+    "approvers": ["@deads2k", "@lavalamp"],
+    "editor": "@sttts",
+    "creationDate": "2019-04-26",
+    "lastUpdated": "2019-07-29",
+    "status": "implemented",
+    "seeAlso": [
+      "/keps/sig-api-machinery/20180731-crd-pruning.md",
+      "/keps/sig-api-machinery/20190425-structural-openapi.md"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Defaulting for Custom Resources\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Validation](#validation)\n  - [Behaviour of Embedded Resource ObjectMeta Defaults](#behaviour-of-embedded-resource-objectmeta-defaults)\n  - [Examples](#examples)\n- [References](#references)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Alternatives considered](#alternatives-considered)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nDefaulting is a fundamental step in the processing of API objects in the request pipeline of the kube-apiserver. Defaulting happens during deserialization, i.e. after decoding of a versioned object, **but before** conversion to a hub type.\n\nDefaulting is implemented for most native Kubernetes API types and plays a crucial role for API compatibility when adding new fields. CustomResources do not support this natively. \n\nMutating admission webhooks can be used to partially replicate defaulting for incoming request payloads. But mutating admission webhooks do not run when reading from etcd.\n\nThis KEP is about adding support for specifying default values for fields via OpenAPI v3 validation schemas in the CRD manifest. OpenAPI v3 has native support for a `default` field with arbitrary JSON values, for example: \n\n```yaml\nproperties:\n  foo:\n    type: string\n    default: \"abc\" \n```\n\nThis KEP proposes to apply these default values during deserialization, in the same way as native resources do. We assume _structural schemas_ as defined in [KEP Vanilla OpenAPI Subset: Structural Schema](/keps/sig-api-machinery/20190425-structural-openapi.md).\n\nThis feature starts behind a feature gate `CustomResourceDefaulting`, disabled by default as alpha in 1.15.\n\nIt will graduate to GA in `apiextensions.k8s.io/v1` for 1.16. Defaults can only be set on creation via the v1 API.\n\n## Motivation\n\n* By far most native Golang based resources implement defaulting. CRDs do not allow that, leading to unnatural, not Kubernetes-like APIs. This is bad for the ecosystem.\n* Mutating Admission Webhooks can be used for defaulting, but:\n  * they are not adequate as it is not possible to set default values of newly added fields on GET because admission is not run in the storage layer when reading from etcd.\n  * webhooks are complex, both from the development/maintenance point of view and due to their non-trivial operational overhead. This makes them a no-go for many \"not so ambitiously, professionally developed CRDs\", e.g. in in-house enterprise environments.\n* _Structural schemas_ as defined in [KEP Vanilla OpenAPI Subset: Structural Schema](/keps/sig-api-machinery/20190425-structural-openapi.md) make defaulting a low hanging fruit.\n \n### Goals\n\n* add CustomResource defaulting at the correct position in the request pipeline \n* allow definition of defaults via the OpenAPI v3 `default` field.\n\n### Non-Goals\n\n* allow non-constant defaults: native Golang code can of course set defaults which depends on other fields of a JSON object. This is out of scope here and would need some kind of defaulting webhook.\n* native-type declarative defaulting: this KEP is about CRDs. Though it might be desirable to support the same `// +default=\u003csome-json-value\u003e` tag and a mechanism in `k8s.io/apiserver` to evaluate defaults in native, generic registries, this is out-of-scope of the KEP though.\n\n## Proposal\n\nWe assume the CRD has _structural schemas_ (as defined in [KEP Vanilla OpenAPI Subset: Structural Schema](/keps/sig-api-machinery/20190425-structural-openapi.md)).\n\nWe propose to\n\n1. derive the value-validation-less variant of the structural schema (trivial by definition of _structural schema_) and \n2. recursively follow the given CustomResource instance and the structural schema, applying specified defaults where an object field is undefined (`if _, ok := obj[field]; !ok` =\u003e default).\n\nThis means that we do not default \n\n* explicit JSON `null` values (it might be rejected during validation depending on the `nullable` setting)\n* nor empty lists or maps\n* nor zero numbers/integers or empty strings (`omitempty` during marshalling indirectly controls whether these values are defaulted or not)\n\nThis corresponds to the [official OpenAPI v3 `default` semantics](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.0.md#properties).\n\nWe do defaulting in the serializer by passing a real defaulter to [`versioningserializer.NewCodec`](https://github.com/kubernetes/apimachinery/blob/master/pkg/runtime/serializer/versioning/versioning.go#L49) such that defaulting is done natively just after the binary payload has been unmarshalled into an `map[string]interface{}` and pruning of [KEP: Pruning for CustomResources](/keps/sig-api-machinery/20180731-crd-pruning.md) was done.\n\nLike for native resources, we do defaulting\n\n* during request payload deserialization\n* after mutating webhook admission\n* during read from storage.\n     \nNote: like for native resources, we do not default after webhook conversions. Hence, webhook conversions should be complete in the sense that they return defaulted objects in order for the API user to see defaulted objects. Technically we could do additional defaulting, but to match native resources, we do not.\n\nCompare the yellow boxes in the following figure:\n\n![Decoding steps which must apply defaults](20190426-crd-defaulting-pipeline.png)\n\nWe rely on the validation steps in the request pipeline to verify that the default value validates value validation. We will check the types in default values using the _structural schema_ during CRD creation and update though. We will also reject defaults which contain values which will be pruned.\n\nDefaulting happens top-down, i.e. we apply defaults for an object first, then dive into the fields (including, the new one).\n\nThe `default` field in the CRD types is considered alpha quality. We will add a `CustomResourceDefaulting` feature gate. Values for `default` will be rejected if the gate is not enabled and there have not been `default` values set before (ratcheting validation). \n\n[Kubebuilder's crd-gen](https://github.com/kubernetes-sigs/controller-tools/tree/master/pkg/crd) can make use of this feature by adding another tag, e.g. `// +default=\u003carbitrary-json-value\u003e`. Defaults are arbitrary JSON values, which must also validate (types are checked during CRD creation and update, value validation is checked for requests, but not for etcd reads) and are not subject to pruning (defaulting happens after pruning).\n\n### Validation\n\nCRDs with defaults can only be created via `apiextensions.k8s.io/v1`. They are rejected for `v1beta1` on creation (updates keep working). \n\nDefault values must be pruned, i.e. must not have fields which are not specified by the given structural schema (including support for `x-kubernetes-preserve-unknown-fields` to exclude nodes from being pruned), with the exception of\n \n* defaults inside of `.metadata` of an embedded resource (`x-kubernetes-embedded-resource: true`)\n* defaults which span `.metadata` of an embedded resource.\n\nValues conflicting with this are rejected when creating or updating a CRD. \n\nDefaults are validated against the schema (including embedded `ObjectMeta` validation).\n\nDefaults inside `.metadata` at the root are not allowed.\n\n### Behaviour of Embedded Resource ObjectMeta Defaults\n\nWhile defaults for `ObjectMeta` fields are not checked for being pruned during CRD creation and update (previous section), we do pruning of default values on CR storage creation. That way no user-provided default values are lost, but the pruning step during storage creation ensures that no unknown or invalid defaulted values are persisted during CR creation or update.\n\n### Examples\n\n1. default in the undefined case\n\n   ```yaml\n   type: object\n   properties:\n     foo:\n       type: string\n       default: \"abc\"\n   ```\n\n   Then\n   \n   ```json\n   {}\n   ```\n   \n   is defaulted to:\n   \n   ```json\n   {\n     \"foo\": \"abc\"\n   }\n   ```\n   \n2. no defaulting\n\n   ```yaml\n   type: object\n   properties:\n     foo:\n       type: string\n       default: \"abc\"\n   ```\n\n   Then\n   \n   ```json\n   {\n     \"foo\": \"def\"\n   }\n   ```\n   \n   is defaulted to:\n   \n   ```json\n   {\n     \"foo\": \"def\"\n   }\n   ```\n   \n3. array default in the undefined case\n\n   ```yaml\n   type: object\n   properties:\n     foo:\n       type: array\n       items:\n         type: integer\n       default: [1]\n   ```\n\n   Then\n   \n   ```json\n   {}\n   ```\n   \n   is defaulted to:\n   \n   ```json\n   {\n     \"foo\": [1]\n   }\n   ```\n   \n   In contrast\n   \n   ```json\n   {\n     \"foo\": null  \n   }\n   ```\n   \n   is defaulting to\n   \n   ```json\n   {\n     \"foo\": null  \n   }\n   ```\n   \n   and then rejected by validation because `foo` has no `nullable: true`.\n   \n   Similarly, empty lists stay empty lists:\n   \n    ```json\n    {\n      \"foo\": [] \n    }\n    ```\n         \n    is defaulted to:\n         \n    ```json\n    {\n       \"foo\": []\n    }\n    ```\n   \n4. top-down defaulting\n      \n   ```yaml\n   type: object\n   properties:\n     foo:\n       type: object\n       properties:\n         a:\n           type: string\n           default: \"abc\"\n         b:\n           type: string\n       default: {\"b\":\"def\"}\n   ```\n\n   Then\n   \n   ```json\n   {}\n   ```\n   \n   is defaulted to:\n   \n   ```json\n   {\n     \"foo\": {\"a\":\"abc\", \"b\":\"def\"}\n   }\n   ```\n\n## References\n\n* Old pruning implementation PR https://github.com/kubernetes/kubernetes/pull/64558, to be adapted. With structural schemas it will become much simpler.\n* [OpenAPI v3 specification](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.0.md)\n* [JSON Schema](http://json-schema.org/)\n\n### Test Plan\n\n**blockers for alpha:**\n\n* we add unit tests for the general defaulting algorithm\n* we add apiextensions-apiserver integration tests to\n  * verify that CRDs with default, but without structural schemas are rejected by validation.\n  * verify that CRDs without defaults keep working (probably nothing new needed)\n  * verify that CRDs with defaults are defaulted if the feature gate is enabled:\n    * during request payload deserialization\n    * during mutating webhook admission\n    * during read from storage\n  * verify with feature gate closed that CRDs with defaults are rejected on create and on updating for the first default value, but accepted if defaults existed before (ratcheting validation).\n\n**blockers for beta:**\n\n* add tests for default value validation:\n  * CRD schema with defaults containing unknown fields inside metadata\n      * new CRD (allowed)\n      * updated CRD where existing CRD did contain the unknown field (allowed)\n      * updated CRD where existing CRD did not contain the unknown field (allowed)\n      * building CR storage from persisted CRD discards unknown fields\n  * CRD schema with defaults containing schema-invalid fields inside metadata (e.g. finalizers: 1)\n    * new CRD (forbidden)\n    * updated CRD where existing CRD did contain the invalid field (allowed)\n    * updated CRD where existing CRD did not contain the invalid field (forbidden)\n    * building CR storage from persisted CRD discards schema invalid fields from defaults.\n  * CRD schema with defaults containing unknown fields outside of metadata is\n    * accepted if they fall under the scope of `x-kubernetes-preserve-unknown-fields: true`\n    * rejected otherwise.\n\n* we are happy with the API and its behaviour\n\n**blockers for GA:**\n* add tests for default value validation:\n  * CRD schema with defaults is rejected on creation via the `v1beta1` endpoints, but updates via `v1beta1` are still possible.\n* we verified that performance of defaulting is adequate and not considerably reducing throughput. As a rule of thumb, we expect defaulting to be considerably faster than a deep copy.\n\n### Graduation Criteria\n\n* the test plan is fully implemented for the respective quality level\n\n### Upgrade / Downgrade Strategy\n\n* defaults cannot be set in 1.14 CRDs.\n* CRDs created in 1.15 will keep defaults when downgrading to 1.14 (because we have them in our `v1beta1` types already). They won't be effective and the CRD will not validate anymore. This is acceptable for an alpha feature.\n* CRDs created in 1.15 with the feature gate enabled will keep working the same way when upgrading to 1.16, or conversely during downgrade from 1.16 to 1.15 as we do ratcheting validation.\n* Creation of CRDs with defaults via `v1beta1` API will be disabled, even if the feature gate is explicitly enabled.\n\n### Version Skew Strategy\n\n* kubectl is not aware of defaulting in any relevant way. \n\n## Alternatives considered\n\n* we considered following the behaviour of native resources regarding `null` values and empty lists/maps and zero values for string, number and integer, i.e. this semantics:\n\n  For atomic types:\n     \n  * `if v, ok := obj[fld]; !ok` =\u003e default\n  * `else if !nullable \u0026\u0026 v == nil` =\u003e default\n     \n  and for `array` type in the schema one of these:\n     \n  * `if v, ok := obj[fld]; !ok` =\u003e default\n  * `else if !nullable \u0026\u0026 v == nil` =\u003e default\n  * `else if array, ok := v.([]interface{}); !ok` =\u003e return deserialization error\n  * `else if !nullable \u0026\u0026 len(array) == 0` =\u003e default\n    \n  and for `object` type in the schema:\n    \n  * `if v, ok := obj[fld]; !ok` =\u003e default\n  * `else if !nullable \u0026\u0026 v == nil` =\u003e default\n  * `else if object, ok := v.(map[string]interface{}); !ok` =\u003e return deserialization error\n  * `else if !nullable \u0026\u0026 len(object) == 0` =\u003e default.\n\n  We decided against that because\n  \n  1. the native-type defaulting semantics inherited\n  \n     * the Golang unmarshalling behaviour which identifies undefined and `null` values (if one does not use additional pointer types to fight against it)\n     * the Protobuf inability to distinguish undefined and empty lists and maps.\n     \n     For CRDs we can distinguish undefined and `null`. We control Protobuf for CRDs, when we add support in the future, and hence can use [some kind of value packaging](https://github.com/protocolbuffers/protobuf/issues/1606#issuecomment-281832148) to represent undefined lists/maps and other values faithfully.\n     \n     This way we avoid cargo culting an accidental API convention.\n     \n  2. the semantics of \"default if undefined\" is much simpler than any variant with (potentially conditional) defaulting of `null`  and empty slices/maps.\n         \n  3. it conflicts with [official OpenAPI v3 semantics for `default`](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.0.md#properties)\n  \n     If we need more native-type like defaulting in addition, we can add an alternative\n      \n     ```yaml\n     x-kubernetes-legacy-default: \u003carbitrary-json\u003e\n     ``` \n\n     which is mutual exclusive with `default`. Using another field name avoids this conflict, but gives us the legacy behaviour of defaulting `null` and empty lists/maps.\n     \n     This might potentially be relevant when embedding upstream Kubernetes types into CRDs.\n\n## Implementation History\n"
+  },
+  {
+    "id": "7def8e64749f0b1355f9309a9e4a0275",
+    "title": "Immutable Fields",
+    "authors": ["@apelisse", "@sttts"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["sig-api-machinery"],
+    "reviewers": ["@erictune", "@jpbetz", "@liggitt", "@logicalhan", "@p0lyn0mial"],
+    "approvers": ["@liggitt", "@deads2k"],
+    "editor": "@sttts",
+    "creationDate": "2019-06-03",
+    "lastUpdated": "2019-10-01",
+    "status": "provisional",
+    "seeAlso": ["/keps/sig-api-machinery/0006-apply.md"],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Immutable fields\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [OpenAPI extension \u003ccode\u003ex-kubernetes-immutable\u003c/code\u003e](#openapi-extension-)\n  - [OpenAPI extension \u003ccode\u003ex-kubernetes-immutable-keys\u003c/code\u003e](#openapi-extension--1)\n  - [Publishing](#publishing)\n  - [Suggested marker syntax](#suggested-marker-syntax)\n  - [Future outline sketch: native resources](#future-outline-sketch-native-resources)\n  - [Future outline: protobuf](#future-outline-protobuf)\n  - [Mutating admission chain](#mutating-admission-chain)\n  - [Where does this happen](#where-does-this-happen)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Alternative Considered](#alternative-considered)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nA lot of fields in APIs tend to be \"immutable\", they can't be changed after\ncreation. This is true for example for many of the fields in pods. There is\ncurrently no way to declaratively specify that fields are immutable, and one\nhas to rely on either built-in validation for core types, or have to build a\nvalidating webhooks for CRDs.\n\nProviding a new `// +immutable` marker would help \n- to make the API more descriptive to users\n- to help API developers by verifying these assertions automatically\n- and to publish this information via OpenAPI.\n\n## Motivation\n\nThere are resources in Kubernetes which have immutable fields by design,\ni.e. after creation of an object, those fields cannot be mutated anymore. E.g. a\npod's specification is mostly unchangeable once it is created. To change the\npod, it must be deleted, recreated and rescheduled. Users want to implement the\nsame kind of read-only semantics for CustomResources, for example:\nhttps://github.com/kubernetes/kubernetes/issues/65973. Today this is only possible\nwith the unreasonable development overhead of a webhook.\n\n### Goals\n\n- extend the CRD API to be able to specify immutability for fields.\n- publish the immutability field of CRDs via OpenAPI as vendor extension.\n- verify immutability on CR update and patch requests.\n- propose a source code marker to be consumed by kubebuilder and openapi-gen.\n- the semantics of immutability must be driven by:\n  - we do not break/change old CRD persistence semantics.\n  - the user-observed equality used for immutability checks must match the equality on\n    persisted objects. I.e. if `StorageRoundtrip(object)` is the object returned by a\n    create or update call, then we want that `StorageRoundtrip(a) == StorageRoundtrip(b)`\n    is the equality used for comparing `a` and `b` **only modulo the order in `x-kubernetes-list-type: set` arrays**. If that check fails, a request\n    is rejected because of immutability conflict.\n- the mechanism must extend to\n  - the addition of protobuf or other encodings which unify values like empty, null and undefined.\n  - the use for existing native types in order to replace complex validation code with a simple declarative marker on the types.\n  - the restriction of the equality to only map keys, but not their values.\n  - the allowance of addition and/or deletion of map keys.\n  - the allowance of addition and/or deletion of keys in array of list-type `map`.  \n\n### Non-Goals\n\n- The mechanism must be extensible to native types, but its implementation is optional.\n- The mechanism must be extensible to future normalization behaviours which will be\n  required to support protobuf for CRs. But this KEP does not aim at defining these\n  and hence defining a custom equality which is compatible with normalization.\n- The mechanism is not supposed to allow different orders in lists to be considered equal.\n\n## Proposal\n\nWe propose\n\n1. adding boolean vendor extensions to CRD OpenAPI schemas named `x-kubernetes-immutable` and `x-kubernetes-immutable-keys` with `true` as the only valid value.\n2. do **strict deep-equal** comparison of those fields marked as immutable during\n   update validation in the request pipeline, with these exceptions:\n   - for fields with `x-kubernetes-immutable-keys: true` only the keys of the map or array (specified via `x-kubernetes-list-map-keys`) are compared.\n   - for fields with `x-kubernetes-list-type: set` the order is ignored.\n   \n   If that comparison fails, the request is rejected as 400 \"Bad Request\".\n   \nWe create another KEP to define custom normalization steps for CRs done during \ndeserialization from etcd and when receiving a request (just after pruning and defaulting).\n\n### OpenAPI extension `x-kubernetes-immutable`\n\nThe `x-kubernetes-immutable` vendor extension is set in the spec of fields,\narrays and objects. It recursively applies to the whole subtree:\n\n```yaml\nproperties:\n  foo:\n    type: array\n    items:\n      x-kubernetes-immutable: true\n      type: string\n```\n\nmeans that the `foo` array itself is mutable, but the items are not. This means that\nitems can be deleted and added, but at each existing index the string values are\nimmutable.\n\n```yaml\nproperties:\n  foo:\n    type: object\n    x-kubernetes-immutable: true\n```\n\nmeans that the whole `.foo` object is immutable. It cannot be removed or set.\n\nIf one wants to make only field immutable, but allow to delete an existing object\nor set it if unset, the following is used:\n\n```yaml\nproperties:\n  foo:\n    type: object\n    properties:\n      \"x\":\n        type: string\n        x-kubernetes-immutable: true\n      \"y\":\n        type: string\n        x-kubernetes-immutable: true\n```\n\nfor all fields `x` and `y` of `foo`.\n\n```yaml\nproperties:\n  foo:\n    type: object\n    x-kubernetes-immutable: true\n    properties:\n      bar:\n        type: object\n        x-kubernetes-immutable: true\n```\n\ncan be simplified to \n\n```yaml\nproperties:\n  foo:\n    type: object\n    x-kubernetes-immutable: true\n    properties:\n      bar:\n        type: object\n```\n\nAt the root-level and inside `.metadata` of an object, `x-kubernetes-immutable` is forbidden.\n\n### OpenAPI extension `x-kubernetes-immutable-keys`\n\nThe topology of arrays is specified via `x-kubernetes-list-type` and `x-kubernetes-list-map-keys`. We currently support the `atomic`, `set` and `map` types. These interact with immutability in the following way:\n\n- **Non-recursive** immutability: For an array or a map, one can not remove or add new items/values, but existing items/values can be modified. \n\n  - For maps this means that the the keys stay the same, but values might change:\n    ```yaml\n    properties:\n      someMap:\n        type: object\n        x-kubernetes-immutable-keys: true # allowed with additionalProperties\n        additionalProperties:\n          type: string\n    ```\n    with `x-kubernetes-immutable-keys` allowed due to `additionalProperties` and mutually\n    exclusive with `x-kubernetes-immutable` on the same node.\n    \n  - For arrays of list-type `map` (associative lists) we get the same behaviour, but enforce that the map-keys\n    are immutable as well (via CRD validation):\n    \n    ```yaml\n    properties:\n      someArray:\n        type: array\n        x-kubernetes-list-type: map\n        x-kubernetes-list-map-keys: [\"name\"]\n        x-kubernetes-immutable-keys: true # allowed with `x-kubernetes-list-type: map`\n        items:\n          properties:\n            name:\n              type: string\n              x-kubernetes-immutable: true # enforced\n            x:\n              type: string\n            y:\n              type: integer\n     ```\n     with `x-kubernetes-immutable-keys` allowed due to `x-kubernetes-list-type: map`.\n     \n  In any other context, without the concept of keys, we do not allow `x-kubernetes-immutable-keys` to be set. Especially for `x-kubernetes-list-type: set`, by definition the whole list items behave as keys and therefore `x-kubernetes-immutable: true` and `x-kubernetes-immutable-keys: true` coincide. We choose to only allow `x-kubernetes-immutable: true` in that case.\n  \n- **Allowed addition and deletion**: New/deleted keys in maps are tolerated.\n\n  - For maps, this can be expressed by marking the values immutable,\n    but not the map:\n    ```yaml\n    properties:\n      someMap:\n        type: object\n        additionalProperties:\n          type: string\n          x-kubernetes-immutable: true\n    ```\n  - For arrays an identifier field is required, e.g. `.name`. We support this by reusing `x-kubernetes-list-type: map` and `x-kubernetes-list-map-keys` specifying the keys:\n    ```yaml\n    properties:\n      someArray:\n        type: array\n        x-kubernetes-list-type: map\n        x-kubernetes-list-map-keys: [\"name\"]\n        items:\n          x-kubernetes-immutable: true\n          properties:\n            name:\n              type: string\n            x:\n              type: string\n            y:\n              type: integer\n    ```\n- **Immutable sets**: For arrays of list-type `set` the equality ignores the order:\n\n    ```yaml\n    properties:\n      someSet:\n        type: array\n        x-kubernetes-list-type: set\n        x-kubernetes-immutable: true\n        items:\n          type: object\n          properties:\n            x:\n              type: string\n            y:\n              type: integer\n    ```\n    such that `{\"someSet\":[{\"x\":\"abc\"},{\"x\":\"def\",\"y\":1}]}` and `{\"someSet\":[{\"x\":\"def\",\"y\":1},{\"x\":\"abc\"}]}` are considered equally and not mutated.\n    \n- (Possible future extension) **Addition only/Deletion only**: Maps and arrays of list-type `map` and `set` can be addition-only and deletion-only by setting either `x-kubernetes-addition-only: true` or `x-kubernetes-deletion-only: true`, both mutually exclusive with `x-kubernetes-immutable` and `x-kubernetes-immutable-keys`.\n  \n  - For maps:\n    ```yaml\n    properties:\n      someMap:\n        type: object\n        x-kubernetes-addition-only: true # allowed with additionalProperties\n        additionalProperties:\n          type: string\n    ```\n    and similarly for `x-kubernetes-deletion-only: true`.\n      \n  - For arrays of list-type `map` we get the same behaviour, but enforce that the map-keys\n    are immutable as well (via CRD validation):\n      \n    ```yaml\n    properties:\n      someArray:\n        type: array\n        x-kubernetes-list-type: map\n        x-kubernetes-list-map-keys: [\"name\"]\n        x-kubernetes-addition-only: true # allowed with `x-kubernetes-list-type: map`\n        items:\n          properties:\n            name:\n              type: string\n              x-kubernetes-immutable: true # enforced\n            x:\n              type: string\n            y:\n              type: integer\n    ```\n    and similarly for `x-kubernetes-deletion-only: true`.\n      \n  - For arrays of list-type `set` get:\n  \n    ```yaml\n    properties:\n      someSet:\n        type: array\n        x-kubernetes-list-type: set\n        x-kubernetes-addition-only: true # allowed with `x-kubernetes-list-type: set`\n        items:\n          type: object\n          properties:\n            x:\n              type: string\n            y:\n              type: integer\n    ```\n    and similarly for `x-kubernetes-deletion-only: true`.\n    \nFor list type `atomic` we disallow `x-kubernetes-immutable-keys`.\n\nNote: it is planned to add `x-kubernetes-map-type` as the equivalent vendor extensions for map to distinguish between classical and atomic maps with the values `map` and `atomic`. The `map` case is the same as the one without any specified map type, and for atomic maps we disallow `x-kubernetes-immutable-keys` again.\n    \n### Publishing\n\nThe `x-kubernetes-immutable` and `x-kubernetes-immutable-keys` vendor extensions are published via `/openapi/v2` as is.\n\n### Suggested marker syntax\n\nIn analogy to `+required`, `+optional` we propose to add a marker to kubebuilder's\ncontroller-gen named `+immutable`, which covers the whole subtree of the object \n(compare field-selection section for extension).\n\n```\n// The name can not be changed after creation.\n// +immutable\nName string\n\n// The list of containers can not change AT ALL after creation.\n// No single field in existing containers can be changed, added or deleted,\n// no new containers can be added, no existing container can be removed.\n// +immutable\nContainers []Containers\n```\n\nThe `x-kubernetes-immutable-keys: true` vendor extension is expressed via `// +immutable=keys` in Golang types:\n\n### Future outline sketch: native resources\n\nFor native resources we add a pre-immutability-check normalization step for objects \ndecoded from JSON which have normalizations defined:\n\n1. versioned JSON blob comes in a request\n2. unmarshalled into versioned Golang struct\n3. defaulting\n4. conversion to internal\n5. if immutability is enabled for the resource:\n   1. marshalling into JSON in-memory\n   2. normalization creating a copy with shared data-structures\n   3. strict immutability check against the old object, coming from proto assuming it is normalized.\n\n### Future outline: protobuf\n\nProtobuf encoding unifies unset, empty and null values for slices and maps. Those three cases \ncannot be differentiated an lead to the same on-the-wire value. In native types\nwe use a semantic equality to factor those differences out if JSON is used on the wire, but\nprotobuf is used for storage. In CRs decoding and encoding is always faithful with the \ntraditional JSON format used on-the-wire and for storage.\n\nWhen adding protobuf encoding to CRDs in the future, we have to (without major, non-standard\nefforts) identify unset, empty and null for CRs as well. This leads to the idea to use a less\nstrict equality in the immutabiltiy checks with that case in mind. But protobuf encoding in\nnative types also has a normalization effect, namely posted JSON object are normalized through\nthe encoding to protobuf when writing to etcd. \n\nHence, it looks sensible to split normalization from immutability equality, and keep a strict \ndeep-equal equality even for protobuf, and potentially native types (if we decide to implement \nimmutability for those).\n\nWith the container example we get this, with a sketch of a `+normalize` marker which\nnormalizes empty and null to undefined:\n\n```\n// The list of containers can not change AT ALL after creation, modulo\n// empty, null and undefined.\n// No single field in existing containers can be changed, added or deleted,\n// no new containers can be added, no existing container can be removed.\n// +immutable\n// +normalize=undefined\nContainers []Container `json:containers,omitempty` `protobuf:\"bytes,2,opt,name=containers\"\n```\n\nFor fields which carry no `omitempty`, we could allow more advanced normalization\nmodes which replicate the Golang serialization behaviour. Tooling like openapi-gen\nand the CRD validation could verify that the normalization specification matches \nthat of Golang.\n\n### Mutating admission chain\n\nMutating admission chain would have the exact same effects as user changes,\nmeaning that they wouldn't be able to change an object after creation. This is\nvery similar to what is done today since validation for updates is run AFTER all\nmutations.\n\n### Where does this happen\n\nThis process is meant to happen right before the update validation and after\nmutating, but before validating webhooks, and only run on updates. This will allow us to\nkeep the exact same behavior while removing the validation code that checks the\nimmutability of fields.\n\n![Decoding steps which must apply defaults](20191001-crd-immutability-pipeline.png)\n\n### Risks and Mitigations\n\n- immutable metadata would break API machinery. We forbid the `x-kubernetes-immutable`\n  at the root of the object and inside `.metadata`. `kind` and `apiVersion` are\n  immutable implicitly. We might publish immutable though for some of these fields.\n\n## Design Details\n\n### Test Plan\n\n- exhaustive unit tests are added in apiextensions-apiserver for\n  - CRD validation\n    - for `x-kubernetes-immutable: true` and `x-kubernetes-immutable-keys: true` at the root and in `metadata`\n    - for `x-kubernetes-immutable-keys: true` for list-type `map` and forbidden otherwise\n    - for `x-kubernetes-immutable: true` on the list-keys of `x-kubernetes-immutable-keys: true` maps.\n    - for `x-kubernetes-immutable: true` and `x-kubernetes-immutable-keys: true` only for v1 CRDs, or during ratcheting updates.\n  - immutability checking with all variants of `x-kubernetes-immutable`, `x-kubernetes-immutable-keys` and `x-kubernetes-list-type`.\n- integration tests are added for\n  - creation, updates, patches and server-side-apply of partially immutable CRs \n  - interaction of server-side-apply list-types and immutability\n  - OpenAPI publishing of the vendor extensions\n  - CRD updates of the immutability extensions and that the new immutability\n    schemas are followed.\n  - rejection of `x-kubernetes-immutable: true` and `x-kubernetes-immutable-keys: true` for non-v1 CRDs\n- e2e and conformance tests that\n  - immutability is followed during updates, patches and server-side-apply. \n   \n### Graduation Criteria\n\nBecause of the very limited risk of the additional immutability check and our\nexperience with defaulting getting no additional soak time by the community\nwe propose to start directly as beta.\n\nFor beta:\n\n- performance does not suffer for CRDs **which do not use** immutability vendor extensions.\n\nFor GA:\n\n- performance is benchmarked with an upper bound overhead of 15% on CRDs with schemas.\n\n## Implementation History\n\nN/A\n\n## Alternative Considered\n\n- OpenAPI has a notion of `readOnly`. This is meant to restrict fields to be set\n  only in responses, not in a request payload. This does not match our \n  `never-change-after-creation` semantics.\n- Allowing `false` as value for `x-kubernetes-immutable: false` was considered to\n  disable immutability imposed by a parent node. This complicates the semantics\n  considerably and can be expressed with a combination of `x-kubernetes-immutable-keys` \n  and `x-kubernetes-immutable` on the complementing fields.\n"
+  },
+  {
+    "id": "2cdcff40b629fd304851d9945c95b3c6",
+    "title": "OwnerReference Resource Field",
+    "authors": ["@deads2k"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["sig-api-machinery"],
+    "reviewers": ["@sttts", "@caesarxuchao", "@liggitt"],
+    "approvers": ["@caesarxuchao", "@lavalamp"],
+    "editor": "@deads2k",
+    "creationDate": "2019-06-07",
+    "lastUpdated": "2019-06-12",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# OwnerReference Resource Field\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Behavior of new clusters](#behavior-of-new-clusters)\n  - [Behavior of old clusters](#behavior-of-old-clusters)\n  - [Behavior with old clients](#behavior-with-old-clients)\n- [References](#references)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Alternatives considered](#alternatives-considered)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nOwnerReferences are used to track garbage collection references.  Today they use `kind`.  This is a serialization format\nnot a URL to access on the kube-apiserver.  To find a URL, a series of upwards of 20 discovery calls are made to find\nevery resource in the cluster to find a list of potential resources (URLs more or less).  This approximates user intent\nand mostly works, but it is inefficient and unnecessarily vulnerable to network outages.\n\nThis KEP proposes adding an optional `resource` field to OwnerReferences.  If provided, this will be the authoritative value and\nno lookup or approximation of user intent will be required.  If not provided, the `kind` will be used as it is today.\nIf both are provided, old clusters continue to work and new clusters will be more efficient and resilient.\n \n```yaml\nownerReferences:\n- apiVersion: apps/v1\n  kind: DaemonSet\n  resource: daemonsets\n```\n\n## Motivation\n\n* Allow precision.  `kind` is a one to many relationship that makes the exact object reference inefficient or impossible to determine.\n* Remove remote calls.  Because a `kind` to `resource` mapping requires discovery, it is vulnerable to network segmentation\n \n### Goals\n\n* Improve reliability. \n* Improve efficiency.\n* Improve predictability.\n* Avoid disrupting existing clients.\n\n### Non-Goals\n\n* Improving discovery.  No amount of improving speed can fix predictability and we're already doing what caching is practical.\n* Improve rest mapping.  No amount of feature enhancement can fix predictability.\n\n## Proposal\n\nAdd a new, optional field to `OwnerReferences` that represents the `resource`.\n\n```go\ntype OwnerReference struct {\n\t// Kind of the referent.\n\tKind string `json:\"kind\" protobuf:\"bytes,1,opt,name=kind\"`\n\t// Resource of the referent.\n\t// +optional\n\tResource string `json:\"resource,omitempty\" protobuf:\"bytes,8,opt,name=resource\"`\n}\n```\n\nThe only impact is in the case of a new cluster with both fields set.  In this case, the new cluster does not have to \nperform any lookups and will trust the client's specification of `resource`.\n\nExisting clients are not required to make this change, but it will improve reliability.  If a client sets the value improperly,\nthen the behavior is congruent to behavior with a mis-set `kind`.  If the kind (resource) is valid, but incorrect \nGC collects the object immediately.  If the kind (resource) is invalid, GC reports the error and waits.  \n\n\n### Behavior of new clusters\n1. `resource` set and `kind` both set: only `resource` is honored and no mapping required  \n2. `resource` set and `kind` not set: validation failure.  Allowing this would break old clusters and clients.  \n3. `resource` not set and `kind` set: `kind` is honored, exactly as before\n4. `resource` not set and `kind` not set: validation failure\n  \n### Behavior of old clusters\n1. `resource` set and `kind` both set: only `kind` is honored, exactly as before  \n2. `resource` set and `kind` not set: validation failure  \n3. `resource` not set and `kind` set: `kind` is honored, exactly as before\n4. `resource` not set and `kind` not set: validation failure\n\n### Behavior with old clients\nOld clients may clear the new `resource` field.  This is most likely to happen with the kube-controller-manager in a downgrade\nscenario, but can logically happen with any client.  In this case, we won't consider the request to be mutating the owner reference\nif there is a valid mapping between the resource and kind.  If there is not such a mapping, we will consider the request to be\nmutating owner reference for the purpose of admission checks. \n\nThere is a case where a controller (not a built in one) running as cluster-admin strips the fields on a blockownerdeletion\nreference.  If this happens, then the check may pass even when it should be rejected.  Because it's a synthetic\npermission against a resource that doesn't exist, this won't be a common risk and the effect is only to prevent deletion.\nNo other escalation or visibility is granted.  We plan to accept this edge and affected projects can update to preserve\nthe data.\n\n## References\n\n* [OwnerReference struct](https://github.com/kubernetes/apimachinery/blob/kubernetes-1.14.4-beta.0/pkg/apis/meta/v1/types.go#L303-L329)\n* [Example mapping in admission](https://github.com/kubernetes/kubernetes/blob/v1.14.4-beta.0/plugin/pkg/admission/gc/gc_admission.go#L184-L187)\n\n### Test Plan\n\n**blockers for GA:**\n\n* kube-controller-manager should set this field.\n* GC admission and controller should consider the field authoritative if present\n\n### Graduation Criteria\n\n* the test plan is fully implemented for the respective quality level\n\n### Upgrade / Downgrade Strategy\n\n* `kind` must always be set.  Validation will ensure this.\n* if `resource` is missing, GC admission and controller must behave exactly as they did before.\n\n### Version Skew Strategy\n\n* unknown fields are dropped server-side, so there is no impact\n* unknown fields must be ignored client-side, so there is no impact\n\n## Alternatives considered\n\n* make discovery faster.  This doesn't solve the network partition issue.\n* cache discovery longer.  We already have a cache, but it doesn't cover all cases like cold starts and expired data.\n* stop looking for RESTMappings if any match is found.  This results in missing the intended referent in multi-match cases\n and doesn't solve the problem with cold starts and expired data.\n\n## Implementation History\n"
+  },
+  {
+    "id": "e587dbecb3eddffddf42c56f7608bdc9",
+    "title": "k8s.io Group Protection",
+    "authors": ["@deads2k"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["sig-api-machinery", "sig-architecture"],
+    "reviewers": ["@sttts", "@jpbetz", "@liggitt"],
+    "approvers": ["@liggitt", "@sttts"],
+    "editor": "@deads2k",
+    "creationDate": "2019-06-12",
+    "lastUpdated": "2019-07-03",
+    "status": "implemented",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# k8s.io Group Protection\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Behavior of new clusters](#behavior-of-new-clusters)\n  - [Behavior of old clusters](#behavior-of-old-clusters)\n  - [What to do if you accidentally put an unapproved API in a protected group](#what-to-do-if-you-accidentally-put-an-unapproved-api-in-a-protected-group)\n- [References](#references)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Alternatives considered](#alternatives-considered)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nAPI groups are organized by namespace, similar to java packages.  `authorization.k8s.io` is one example.  When users create\nCRDs, they get to specify an API group and their type will be injected into that group by the kube-apiserver.\n\nThe `*.k8s.io` or `*.kubernetes.io` groups are owned by the Kubernetes community and protected by API review (see [What APIs need to be reviewed](https://github.com/kubernetes/community/blob/master/sig-architecture/api-review-process.md#what-apis-need-to-be-reviewed),\nto ensure consistency and quality.  To avoid confusion in our API groups and prevent accidentally claiming a\nspace inside of the kubernetes API groups, the kube-apiserver needs to be updated to protect these reserved API groups.\n\nThis KEP proposes adding an `api-approved.kubernetes.io` annotation to CustomResourceDefinition.  This is only needed if\nthe CRD group is `k8s.io`, `kubernetes.io`, or ends with `.k8s.io`, `.kubernetes.io`.  The value should be a link to a\nto a URL where the current spec was approved, so updates to the spec should also update the URL.\n \n```yaml\nmetadata:\n  annotations:\n    \"api-approved.kubernetes.io\": \"https://github.com/kubernetes/kubernetes/pull/78458\"\n```\n\n## Motivation\n\n* Prevent accidentally including an unapproved API in community owned API groups\n \n### Goals\n\n* Ensure API consistency. \n* Prevent accidentally claiming reserved named.\n\n### Non-Goals\n\n* Prevent malicious users from claiming reserved names.\n\n## Proposal\n\nThis KEP proposes adding an `api-approved.kubernetes.io` annotation to CustomResourceDefinition.  This is only needed if\nthe CRD group is `k8s.io`, `kubernetes.io`, or ends with `.k8s.io`, `.kubernetes.io`.  The value should be a link to the\npull request where the API has been approved.  If the API is unapproved, you may set the annotation to a string starting\nwith `\"unapproved\"`.  For instance, `\"unapproved, temporarily squatting` or `\"unapproved, experimental-only\"`.  This \nis discouraged.\n \n```yaml\nmetadata:\n  annotations:\n    \"api-approved.kubernetes.io\": \"https://github.com/kubernetes/kubernetes/pull/78458\"\n```\n\n```yaml\nmetadata:\n  annotations:\n    \"api-approved.kubernetes.io\": \"unapproved, experimental-only\"\n```\n\nThis field is used by new kube-apiservers to set the `KubeAPIApproved` condition.  \n 1. If a new server sees a CRD for a resource in a kube group and sees the annotation set to a URL, it will set the `KubeAPIApproved` condition to true.\n 2. If a new server sees a CRD for a resource in a kube group and sees the annotation set to `\"unapproved.*\"`, it will set the `KubeAPIApproved` condition to false.\n 3. If a new server sees a CRD for a resource in a kube group and does not see the annotation, it will set the `KubeAPIApproved` condition to false.\n 4. If a new server sees a CRD for a resource outside a kube group, it does not set the `KubeAPIApproved` condition at all.\n\nIn v1, this annotation will be required in order to create a CRD for a resource in one of the kube API groups.  If the `KubeAPIApproved` condition is false, \nthe condition message will include a link to https://github.com/kubernetes/enhancements/pull/1111 for reference.\n\n### Behavior of new clusters\n1. Current CRD for a resource in the kube group already in API is missing valid `api-approved.kubernetes.io` or has set the value to `\"unapproved.*\"` - `KubeAPIApproved` condition will be false.\n2. CRD for a resource in the kube group creating via CRD.v1beta1 is missing valid `api-approved.kubernetes.io` - create as normal.  This ensures compatibility.  `KubeAPIApproved` condition will be false.\n3. CRD for a resource in the kube group creating via CRD.v1 is missing valid `api-approved.kubernetes.io` - fail validation and do not store in etcd.\n4. CRD for a resource outside the kube group creating via CRD.v1 is contains the `api-approved.kubernetes.io` - fail validation and do not store in etcd.\n5. In CRD.v1, remove a required `api-approved.kubernetes.io` - fail validation.\n6. In all versions, any update that does not change the `api-approved.kubernetes.io` will go through our current validation rules.\n \n  \n### Behavior of old clusters\n1.  Nothing changes.  The old clusters will persist and keep the annotation\n\nThis doesn't actively prevent bad actors from simply setting the annotation, but it does prevent accidentally claiming\nan inappropriate name. \n\n### What to do if you accidentally put an unapproved API in a protected group\n1. Get the current state and future changes approved.  For community projects, this is the best choice if the current state\n   is approvable.\n2. If there are structural problems with the API's current state that prevent approval, you have two choices.\n   1. restructure in a new version, maintaining a conversion webhook, and plan to stop serving the old version.  There are\n      some cases where this may not work if the changes are not roundtrippable, but they should be rare.\n   2. restructure in a new API group. There will be no connection to existing data.  This may be disruptive for non-alpha APIs, but these \n      names are reserved and the process of API review has been in place for some time.  The expectation is that this is\n      the exceptional case of an exceptional case.\n3. Indicate that your API is unapproved by setting the `\"api-approved.kubernetes.io\"` annotation to something starting with\n   unapproved.  For instance, `\"unapproved, temporarily squatting` or `\"unapproved, experimental-only\"`.\n\n## References\n\n* [Accidental name in Kanary](https://libraries.io/github/AmadeusITGroup/kanary))\n\n### Test Plan\n\n**blockers for GA:**\n\n* Document in the release notes.  The impact is very low\n\n### Graduation Criteria\n\n* the test plan is fully implemented for the respective quality level\n\n### Upgrade / Downgrade Strategy\n\n* annotations and conditions are always persisted.  If set, they remain consistent.  If unset, they also remain consistent.\n\n### Version Skew Strategy\n\n* annotations and conditions are always persisted.  If set, they remain consistent.  If unset, they also remain consistent.\n\n## Alternatives considered\n\n## Implementation History\n"
+  },
+  {
+    "id": "b55e91df0a55b3bc91181b0f98773b77",
+    "title": "Graduate API gzip compression support to GA",
+    "authors": ["@smarterclayton"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["sig-cli"],
+    "reviewers": ["@lavalamp", "@liggitt"],
+    "approvers": ["@liggitt", "@lavalamp"],
+    "editor": "TBD",
+    "creationDate": "2019-03-22",
+    "lastUpdated": "2019-03-22",
+    "status": "implementable",
+    "seeAlso": ["https://github.com/kubernetes/kubernetes/issues/44164"],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Graduate API gzip compression to GA\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [1.16](#116)\n  - [1.17](#117)\n  - [Implementation Details](#implementation-details)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nKubernetes sometimes returns extremely large responses to clients outside of its local network, resulting in long delays for components that integrate with the cluster in the list/watch controller pattern. Kubernetes should properly support transparent gzip response encoding, while ensuring that the performance of the cluster does not regress for small requests.\n\n## Motivation\n\nIn large Kubernetes clusters the size of protobuf or JSON responses may exceed hundreds of megabytes, and clients that are not on fast local networks or colocated with the master may experience bandwidth and/or latency issues attempting to synchronize their state with the server (in the case of custom controllers). Many HTTP servers and clients support transparent compression by use of the `Accept-Encoding` header, and support for gzip can reduce total bandwidth requirements for integrating with Kubernetes clusters for JSON by up to 10x and for protobuf up to 8x.\n\n### Goals\n\nAllow standard HTTP transparent `Accept-Encoding: gzip` behavior to work for large Kubernetes API requests, without impacting existing Go language clients (which are already sending that header) or causing a performance regression on the Kubernetes apiservers due to the additional CPU necessary to compress small requests.\n\n### Non-Goals\n\n* Support other compression formats like Snappy due to limited client support\n* Compress non-API responses\n* Compress watch responses\n\n## Proposal\n\n### 1.16\n\n* Update the existing incomplete alpha API compression to:\n  * Only occur on API requests\n  * Only occur on very large responses (\u003e128KB)\n* Promote to beta and enable by default since this is a standard feature of HTTP servers\n  * Test at large scale to mitigate risk of regression, tune as necessary\n\n### 1.17\n\n* Promote to GA\n\n\n### Implementation Details\n\nKubernetes has had an alpha implementation of transparent gzip encoding since 1.7. However, this\nimplementation was never graduated because it caused client misbehavior and the issues were not resolved.\n\nAfter reviewing the code, the problems in the prior implementation were that it attempted to globally\nprovide transparent compression as an HTTP middleware component at a much higher level than was necessary.\nThe bugs that prevented enablement involved double compression of nested responses and failures to\ncorrectly handle flushing of lower level primitives. We do not need to GZIP compress all HTTP endpoints\nserved by the Kubernetes API server (such as watch requests, exec requests, OpenAPI endpoints which provide\ntheir own compression). Our implementation may satisfy its goals of reducing latency for large requests if\nwe narrowly scope compression to only those endpoints that need compression.\n\nA further complexity is that the standard Go client library (which Kubernetes has leveraged since 1.0)\nalways requests compression. Performance testing showed that enabling compression for all suitable\nAPI responses (objects returned via GET, LIST, UPDATE, PATCH) caused a significant performance regression\nin both CPU usage (2x) and tail latency (2-5x) on the Kubernetes apiservers. This is due to the additional\nCPU costs for performing compression, which impacts tail latency of small requests due to increased\napiserver load. Since forcing all clients in the ecosystem to disable transparent compression by default\nis impractical and cannot be done in a gradual manner, we need to apply a more suitable heuristic than\n\"did the client request transparent compression\". According to the HTTP spec, a server may ignore an\n`Accept-Encoding` header for any reason, which means we decide *when* we want to compress, not just\nwhether we compress.\n\nThe preferred approach is to only compress responses returned by the API server when encoding objects\nthat are large enough for compression to benefit the client but not unduly burden the server. In general,\nthe target of this optimization is extremely large LIST responses which are usually multiple megabytes\nin size. These requests are infrequent (\u003c1% of all reads) and when network bandwidth is lower than typical\ndatacenter speeds (1 GBps) the benefit in reduced latency for clients outweighs the slightly higher CPU\ncost for compression.\n\nWe experimentally determined a size cut-off for compression that caused no regression on the Kubernetes\ndensity and load tests in either 99th percentile latency or kube-apiserver CPU usage of 128KB, which is\nroughly the size of 50 average pods (2.2kb from a large Kubernetes cluster with a diverse workload). This\nimplementation applies this specific heuristic to the place in the Kubernetes code path where we encode\nthe body of a response from a single input `[]byte` buffer due to how Kubernetes encodes and manages\nresponses, which removes the side-effects and unanticipated complexity in the prior implementation.\n\nGiven that this is standard HTTP server behavior and can easily be tested with unit, integration, and\nour complete end-to-end test suite (due to all of our clients already requesting gzip compression),\nthere is minimal risk in rolling this out directly to GA. We suggest preserving the feature gate so that\nan operator can disable this behavior if they experience a regression in highly-tuned large-scale deployments.\n\n### Risks and Mitigations\n\nThe primary risk is that an operator running Kubernetes very close to the latency and tolerance limits\non a very large and overloaded Kubernetes apiserver who runs an unusually high percentage of large\nLIST queries on high bandwidth networks would experience higher CPU use that causes them to hit a CPU\nlimit. In practice, the cost of gzip proportional to the memory and CPU costs of Go memory allocation\non very large serialization and deserialization grows sublinear, so we judge this unlikely. However,\nto give administrators an opportunity to react, we would preserve the feature gate and allow it to be\ndisabled until 1.17.\n\nSome clients may be requesting gzip and not be correctly handling gzipped responses. An effort should\nbe made to educate client authors that this change is coming, but in general we do not consider\nincorrect client implementations to block implementation of standard HTTP features. The easy mitigation\nfor many clients is to disable sending `Accept-Encoding` (Go is unusual in providing automatic\ntransparent compression in the client ecosystem - many client libraries still require opt-in behavior).\n\n## Graduation Criteria\n\nTransparent compression must be implemented in the more focused fashion described in this KEP. The\nscalability sig must sign off that the chosen limit (128KB) does not cause a regression in 5000 node\nclusters, which may cause us to revise the limit up.\n\n## Implementation History\n\n* 1.7 Kubernetes added alpha implementation behind disabled flag\n* Updated proposal with more scoped implementation for Beta in 1.16 that addresses prior issues\n"
+  },
+  {
+    "id": "6d264a30f101b9c27c1cc08a4a8eee43",
+    "title": "Deprecate and remove SelfLink",
+    "authors": ["@wojtek-t"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["sig-scalability"],
+    "reviewers": ["@liggitt", "@smarterclayton"],
+    "approvers": ["@lavalamp", "@deads2k"],
+    "editor": "@wojtek-t",
+    "creationDate": "2019-07-11",
+    "lastUpdated": "2019-07-24",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Deprecate and Remove SelfLink\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n**ACTION REQUIRED:** In order to merge code into a release, there must be an issue in [kubernetes/enhancements] referencing this KEP and targeting a release milestone **before [Enhancement Freeze](https://github.com/kubernetes/sig-release/tree/master/releases)\nof the targeted release**.\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e., [kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be updated for the enhancement to be released.\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\n`SelfLink` is a URL representing a given object. It is part of `ObjectMeta` and `ListMeta`\nwhich means that it is part of every single Kubernetes object.\n\nThis KEP is proposing deprecating this field and removing it in an year according to our\n`Deprecation policy`.\n\n## Motivation\n\nI haven't heard any really compelling reason for having `SelfLink` field. When modifying or\nreading an object from kube-apiserver, its `Selflink` is set to exactly the URL that was\nused to perform that operation, e.g.\n```\napis/apps/v1/namespaces/default/deployments/deployment/status\n```\nSo in order to get the object, client has to knew that URL anyway.\n\nWhat is more, it leaves out exactly the thing that user can't tell from looking at a stored\nobject, which is what cluster and/or server it came from.\n\nAt the same time, setting this `SelfLink` field:\n- is treated in a very special way in generic-apiserver - it is the only field that is being\nset right before serializing the object (as this is the only place that has all the necessary\ninformation to set it)\n- has non-negligible performance impact - constructing the value performs couple memory\nallocations (and memory allocations are things that have visible impact on Kubernetes\nperformance and scalability)\n\nI propose to remove that field after necessary (long enough) deprecation period.\n\n### Goals\n\n- Eliminate performance impact caused by setting `SelfLink`\n- Simplify the flow of generic apiserver by eliminating modifying objects late in the\nprocessing path.\n\n### Non-Goals\n\n- Introduce location/source-cluster fields to ObjectMeta or ListMeta objects.\n\n## Proposal\n\nIn v1.16, we will deprecate the `SelfLink` field in both `ObjectMeta` and `ListMeta`\nobjects by:\n- documenting in field definition that it is deprecated and is going to be removed\n- adding a release-note about field deprecation\nWe will also introduce a feature gate to allow disabling setting `SelfLink` fields\nand opaque the logic setting it behind this feature gate.\n\nIn v1.20 (12 months and 4 release from v1.16) we will switch off the feature gate\nwhich will automatically disable setting SelfLinks. However it will still be possible\nto revert the behavior by changing value of a feature gate.\n\nIn v1.21, we will get rid of the whole code propagating those fields and fields themselves.\nIn the meantime, we will go over places referencing that field (see below) and get rid\nof those too.\n\n### Risks and Mitigations\n\nThe risk is that some users may significantly rely on this field in a way we are not aware of.\nIn that case, we rely on them start shouting loudly and 4 release before fields removal give\nus time to revisit that decision.\n\n## Design Details\n\nI went through a k/k repo (including its staging repos) and all repos under [kubernetes-client][]\nand this is the list of places that reference `SelfLink` fields (I excluded tests and all places\nin apiserver responsible for setting it):\n\n- [ ] https://github.com/kubernetes/kubernetes/blob/master/pkg/api/ref/ref.go\n  Used for detecting version (which I believe should always be set?).\n- [ ] https://github.com/kubernetes/kubernetes/blob/master/pkg/kubectl/cmd/get/get.go\n  Propagating SelfLink in kubectl get.\n- [ ] https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/config/common.go\n  Doesn't seem to be really used anywhere.\n- [ ] https://github.com/kubernetes/kubernetes/blob/master/pkg/printers/tablegenerator.go\n  Setting SelfLink for table format.\n- [ ] https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/tableconvertor/tableconvertor.go\n  Setting SelfLink in conversion to table format for custom resources.\n- [ ] https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apiserver/pkg/registry/rest/table.go\n  Setting SelfLink in conversion to table format.\n- [ ] staging/src/k8s.io/client-go/tools/reference/ref.go\n  A copy of first item literally.\n\n- [ ] https://github.com/kubernetes-client/java/blob/master/kubernetes/src/main/java/io/kubernetes/client/openapi/models/V1ListMeta.java\n  Setter/getter and part of equals() and hashCode() methods.\n- [ ] https://github.com/kubernetes-client/java/blob/master/kubernetes/src/main/java/io/kubernetes/client/openapi/models/V1ObjectMeta.java\n  Setter/getter and part of equals() and hashCode() methods.\n- [ ] https://github.com/kubernetes-client/csharp/blob/master/src/KubernetesClient/generated/Models/V1ListMeta.cs\n  Setter/getter and constructor.\n- [ ] https://github.com/kubernetes-client/csharp/blob/master/src/KubernetesClient/generated/Models/V1ObjectMeta.cs\n  Setter/getter and constructor.\n- [ ] https://github.com/kubernetes-client/go/blob/master/kubernetes/client/v1_list_meta.go\n  Only part of type definition.\n- [ ] https://github.com/kubernetes-client/go/blob/master/kubernetes/client/v1_object_meta.go\n  Only part of type definition.\n- [ ] https://github.com/kubernetes-client/ruby/blob/master/kubernetes/lib/kubernetes/models/v1_list_meta.rb\n  Setter/getter.\n- [ ] https://github.com/kubernetes-client/ruby/blob/master/kubernetes/lib/kubernetes/models/v1_object_meta.rb\n  Setter/getter.\n- [ ] https://github.com/kubernetes-client/perl/blob/master/lib/Kubernetes/Object/V1ListMeta.pm\n  Seems like setter/getter to me.\n- [ ] https://github.com/kubernetes-client/perl/blob/master/lib/Kubernetes/Object/V1ObjectMeta.pm\n  Seems like setter/getter to me.\n- [ ] https://github.com/kubernetes-client/python/blob/master/kubernetes/client/models/v1_list_meta.py\n  Setter/getter.\n- [ ] https://github.com/kubernetes-client/python/blob/master/kubernetes/client/models/v1_object_meta.py\n  Setter/getter.\n\n[kubernetes-client]: https://github.com/kubernetes-client\n\n### Test Plan\n\nNo new tests will be created - we expect all the tests to be passing at each phase of deprecation\nand after removal of the fields.\n\n### Graduation Criteria\n\nThe whole design is about meeting [Deprecation policy][deprecation-policy] - this doesn't\nrequire more explanation.\n\n[deprecation-policy]: https://kubernetes.io/docs/reference/using-api/deprecation-policy/\n\n### Upgrade / Downgrade Strategy\n\nNo specific strategy is required.\n\n### Version Skew Strategy\n\nAll the references to `SelfLink` should be removed early enough (2 releases before) the field\nitself will be removed.\n\n## Implementation History\n\n2019-07-23: KEP merged.\n2019-07-24: KEP move to implementable.\n"
+  },
+  {
+    "id": "78914cc8d9ad85b1dce92bf0c77b3b5a",
+    "title": "StorageVersion API for HA API servers",
+    "authors": ["@xuchao"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": null,
+    "reviewers": ["@deads2k", "@yliaog", "@lavalamp"],
+    "approvers": ["@deads2k", "@lavalamp"],
+    "editor": "",
+    "creationDate": "2019-08-22",
+    "lastUpdated": "2019-08-22",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# StorageVersion API for HA API servers\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Overview](#overview)\n- [API changes](#api-changes)\n  - [Resource Version API](#resource-version-api)\n- [Changes to API servers](#changes-to-api-servers)\n  - [Curating a list of participating API servers in HA master](#curating-a-list-of-participating-api-servers-in-ha-master)\n  - [Updating StorageVersion](#updating-storageversion)\n  - [Garbage collection](#garbage-collection)\n  - [CRDs](#crds)\n  - [Aggregated API servers](#aggregated-api-servers)\n- [Consuming the StorageVersion API](#consuming-the-storageversion-api)\n- [StorageVersion API vs. StorageVersionHash in the discovery document](#storageversion-api-vs-storageversionhash-in-the-discovery-document)\n- [Backwards Compatibility](#backwards-compatibility)\n- [Graduation Plan](#graduation-plan)\n- [FAQ](#faq)\n- [Alternatives](#alternatives)\n  - [Letting API servers vote on the storage version](#letting-api-servers-vote-on-the-storage-version)\n  - [Letting the storage migrator detect if API server instances are in agreement](#letting-the-storage-migrator-detect-if-api-server-instances-are-in-agreement)\n- [Appendix](#appendix)\n  - [Accuracy of the discovery document of CRDs](#accuracy-of-the-discovery-document-of-crds)\n- [References](#references)\n\u003c!-- /toc --\u003e\n\n## Overview\n\nDuring the rolling upgrade of an HA master, the API server instances may\nuse different storage versions encoding a resource. The [storageVersionHash][]\nin the discovery document does not expose this disagreement. As a result, the\nstorage migrator may proceed with migration with the false belief that all API\nserver instances are encoding objects using the same storage version, resulting\nin polluted migration.  ([details][]).\n\n[storageVersionHash]:https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go#L979\n[details]:https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/35-storage-version-hash.md#ha-masters\n\nWe propose a way to show what storage versions all API servers are using, so\nthat the storage migrator can defer migration until an agreement has been\nreached.\n\n## API changes\n\n### Resource Version API\n\nWe introduce a new API `StorageVersion`, in a new API group\n`internal.apiserver.k8s.io/v1alpha1`.\n\n```golang\n//  Storage version of a specific resource.\ntype StorageVersion struct {\n  TypeMeta\n  // The name is \u003cgroup\u003e.\u003cresource\u003e.\n  ObjectMeta\n  \n  // Spec is omitted because there is no spec field.\n  // Spec StorageVersionSpec\n\n  // API server instances report the version they can decode and the version they\n  // encode objects to when persisting objects in the backend.\n  Status StorageVersionStatus\n}\n\n// API server instances report the version they can decode and the version they\n// encode objects to when persisting objects in the backend.\ntype StorageVersionStatus struct {\n  // The reported versions per API server instance.\n  // +optional\n  ServerStorageVersions []ServerStorageVersion\n  // If all API server instances agree on the same encoding storage version,\n  // then this field is set to that version. Otherwise this field is left empty.\n  // +optional\n  AgreedEncodingVersion string\n\n  // The latest available observations of the storageVersion's state.\n  // +optional\n  Conditions []StorageVersionCondition\n  \n}\n\n// An API server instance reports the version it can decode and the version it\n// encodes objects to when persisting objects in the backend.\ntype ServerStorageVersion struct {\n  // The ID of the reporting API server. \n  // For a kube-apiserver, the ID is configured via a flag.\n  APIServerID string\n\n  // The API server encodes the object to this version when persisting it in\n  // the backend (e.g., etcd).\n  EncodingVersion string\n\n  // The API server can decode objects encoded in these versions.\n  // The encodingVersion must be included in the decodableVersions.\n  DecodableVersions []string\n}\n\n\nconst (\n  // Indicates that storage versions reported by all servers are equal.\n  AllEncondingVersionsEqual StorageVersionConditionType = \"AllEncodingVersionsEqual\"\n)\n\n// Describes the state of the storageVersion at a certain point.\ntype StorageVersionCondition struct {\n\t// Type of the condition.\n\tType StorageVersionConditionType\n\t// Status of the condition, one of True, False, Unknown.\n\tStatus corev1.ConditionStatus\n\t// The last time this condition was updated.\n\t// +optional\n\tLastUpdateTime metav1.Time\n\t// The reason for the condition's last transition.\n\t// +optional\n\tReason string\n\t// A human readable message indicating details about the transition.\n\t// +optional\n\tMessage string\n}\n```\n\n## Changes to API servers\n\nIn this section, we describe how to update and consume the StorageVersion API.\n\n### Curating a list of participating API servers in HA master\n\nAPI servers need such a list when updating the StorageVersion API. Currently,\nsuch a list is already maintained in the \"kubernetes\" endpoints, though it is not\nworking in all flavors of Kubernetes deployments.\n\nWe will inherit the existing [mechanism][], but formalize the API and process in\nanother KEP. In this KEP, we assume all API servers have access to the list of\nall participating API servers via some API.\n\n[mechanism]:https://github.com/kubernetes/community/pull/939\n\n### Updating StorageVersion\n\nDuring bootstrap, for each resource, the API server \n* gets the storageVersion object for this resource, or creates one if it does\n  not exist yet,\n* gets the list of participating API servers,\n* updates the storageVersion locally. Specifically,\n  * creates or updates the .status.serverStorageVersions, to express this API\n    server's decodableVersions and encodingVersion.\n  * removes .status.serverStorageVersions entries whose server ID is not present\n    in the list of participating API servers, such entries are stale.\n  * checks if all participating API servers agree on the same storage version.\n    If so, sets the version as the status.agreedEncodingVersion. If not, sets\n    the status.agreedEncodingVersion to empty. The \"AllEncodingVersionsEqual\"\n    status.condition is updated accordingly as well.\n* updates the storageVersion object, using the rv in the first step\n  to avoid conflicting with other API servers.\n* installs the resource handler.\n\nThe above mentioned process requires an API server to update the storageVersion\nbefore accepting API requests. If we don't enforce this order, data encoded in\nan unexpected version can sneak into etcd. For example, an API server persists a\nwrite request encoded in an obsoleted version, then it crashes before it can\nupdate the storageVersion. The storage migrator has no way to detect this write.\n\nFor the cmd/kube-apiserver binary, we plan to enforce this order by adding a new\nfilter to the [handler chain][]. Before kube-aggregator, kube-apiserver, and\napiextension-apiserver have registered the storage version of the built-in\nresources they host, this filter only allows the following requests to pass:\n1. a request sent by the loopbackClient and is destined to the storageVersion\n   API.\n2. the verb of the request is GET.\n3. the request is for an API that is not persisted, e.g.,\n   SubjectAccessReview and TokenReview. [Here] is a complete list.\n4. the request is for an aggregated API, because the request is handled by the\n   aggregated API server.\n5. the request is for a custom resource, because the apiextension apiserver\n   makes sure that it updates the storage version before it serves the CR (see\n   [CRDs](#crds)).\n\nThe filter rejects other requests with a 503 Service Unavailable response code.\n\n[handler chain]:https://github.com/kubernetes/kubernetes/blob/fc8f5a64106c30c50ee2bbcd1d35e6cd05f63b00/staging/src/k8s.io/apiserver/pkg/server/config.go#L639\n[Here]:https://github.com/kubernetes/kubernetes/blob/709a0c4f7bfec2063cb856f3cdf4105ce984e247/pkg/master/storageversionhashdata/data.go#L26\n\nOne concern is that the storageVersion API becomes a single-point-of-failure,\nthough it seems inevitable in order to ensure the correctness of the storage\nmigration.\n\nWe will also add a post-start hook to ensure that the API server reports not\nready until the storageVersions are up-to-date and the filter is turned off.\n\n### Garbage collection\n\nThere are two kinds of \"garbage\":\n\n1. stale storageVersion.status.serverStorageVersions entries left by API servers\n   that have gone away;\n2. storageVersion objects for resources that are no longer served.\n\nWe can't rely on API servers to remove the first kind of stale entries during\nbootstrap, because an API server can go away after other API servers bootstrap,\nthen its stale entries will remain in the system until one of the other API\nservers reboots.\n\nHence, we propose a leader-elected control loop in API server to clean up the\nstale entries, and in turn clean up the obsolete storageVersion objects. The\ncontrol loop watches the list of participating API servers, upon changes, it\nperforms the following actions for each storageVersion object:\n\n* gets a storageVersion object\n* gets the list of participating API servers,\n* locally, removes the stale entries (1st kind of garbage) in\n  storageVersion.status.serverStorageVersions,\n  * after the removal, if all participating API servers have the same\n    encodingVersion, then sets storageVersion.status.AgreedEncodingVersion and\n    status.condtion. \n* checks if the storageVersion.status.serverStorageVersions is empty,\n  * if empty, deletes the storageVersion object (2nd kind of garbage),\n  * otherwise updates the storageVersion object,\n  * both the delete and update operations are preconditioned with the rv in the\n    first step to avoid conflicting with API servers modifying the object.\n\nAn API server needs to establish its membership in the list of participating API\nservers before updating storageVersion, otherwise the above control loop can\nmistake a storageVersion.status.serverStorageVersions entry added by a new API\nserver as a stale entry.\n\n### CRDs\n\nToday, the [storageVersionHash][] in the discovery document in HA setup can\ndiverge from the actual storage version being used. See the [appendix][] for\ndetails.\n\n[appendix]:#appendix\n[storageVersionHash]:https://github.com/kubernetes/kubernetes/blob/c008cf95a92c5bbea67aeab6a765d7cb1ac68bd7/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go#L989\n\nTo ensure that the storageVersion.status always shows the actual encoding\nversions, the apiextension-apiserver must update the storageVersion.status\nbefore it [enables][] the custom resource handler. This way it does not require\nthe [filter][] mechanism that is used by the kube-apiserver to ensure the\ncorrect order.\n\n[enables]:https://github.com/kubernetes/kubernetes/blob/220498b83af8b5cbf8c1c1a012b64c956d3ebf9b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go#L703\n[filter]:#updating-storageversion\n\n### Aggregated API servers\n\nMost code changes will be done in the generic apiserver library, so aggregated\nAPI servers using the library will get the same behavior.\n\nIf an aggregated API server does not use the API, then the storage migrator does\nnot manage its API.\n\n## Consuming the StorageVersion API\n\nThe consumer of the StorageVersion API is the storage migrator. The storage\nmigrator\n* starts migration if the storageVersion.status.agreedEncodingVersion differs\n  from the storageState.status.[persistedStorageVersionHashes][],\n* aborts ongoing migration if the storageVersion.status.agreedEncodingVersion is\n  empty.\n\n[persistedStorageVersionHashes]:https://github.com/kubernetes-sigs/kube-storage-version-migrator/blob/60dee538334c2366994c2323c0db5db8ab4d2838/pkg/apis/migration/v1alpha1/types.go#L164\n\n## StorageVersion API vs. StorageVersionHash in the discovery document\n\nWe do not change how the storageVersionHash in the discovery document is\nupdated. The only consumer of the storageVersionHash is the storage migrator,\nwhich will convert to use the new StorageVersion API. After the StorageVersion\nAPI becomes stable, we will remove the storageVersionHash from the discovery\ndocument, following the standard API deprecation process.\n\n## Backwards Compatibility\n\nThere is no change to the existing API, so there is no backwards compatibility\nconcern.\n\n## Graduation Plan\n\n* alpha: in 1.17, the StorageVersion API and related mechanism will be feature\n  gated by the `ExposeStorageVersion` flag.\n* beta1 in 1.18, beta2 in 1.19. We make two beta releases to allow more time for\n  feedback.\n* GA in 1.20.\n\n## FAQ\n\n1. Q: if an API server is rolled back when the migrator is in the middle of\n   migration, how to prevent corruption? ([original question][])\n\n   A: Unlike the discovery document, the new StorageVersion API is persisted in\n   etcd and has the resourceVersion(RV) field, so the migrator can determine if\n   the storage version has changed in the middle of migration by comparing the\n   RV of the storageVersion object before and after the migration. Also, as an\n   optimization, the migrator can fail quickly by aborting the ongoing migration\n   if it receives a storageVersion change event via WATCH.\n\n   [original question]:https://github.com/kubernetes/enhancements/pull/1176#discussion_r307977970\n\n## Alternatives\n\n### Letting API servers vote on the storage version\n\nSee [#1201](https://github.com/kubernetes/enhancements/pull/920)\n\nThe voting mechanism makes sure all API servers in an HA cluster always use the\nsame storage version, and the discovery document always lists the selected\nstorage version.\n\nCons:\n* The voting mechanism adds complexity. For the storage migrator to work\n  correctly, it is NOT necessary to guarantee all API server instances always\n  use the same storage version.\n\n### Letting the storage migrator detect if API server instances are in agreement\n\nSee [#920](https://github.com/kubernetes/enhancements/pull/920)\n\nCons: it has many assumptions, see [cons][].\n[cons]:https://github.com/kubernetes/enhancements/pull/920/files#diff-a1d206b4bbac708bf71ef85ad7fb5264R339\n\n## Appendix\n\n### Accuracy of the discovery document of CRDs\n\nToday, the storageVersionHash listed in the discovery document \"almost\"\naccurately reflects the actual storage version used by the apiextension-apiserver.\n\nUpon storage version changes in the CRD spec,\n* [one controller][] deletes the existing resource handler of the CRD, so that\n  a new resource handler is created with the latest cached CRD spec is created\n  upon the next custom resource request. \n* [another controller][] enqueues the CRD, waiting for the worker to updates the\n  discovery document.\n\n[one controller]:https://github.com/kubernetes/kubernetes/blob/1a53325550f6d5d3c48b9eecdd123fd84deee879/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go#L478\n[another controller]:https://github.com/kubernetes/kubernetes/blob/1a53325550f6d5d3c48b9eecdd123fd84deee879/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go#L258\n\nThese two controllers are driven by the [same informer][], so the lag between\nwhen the server starts to apply the new storage version and when the discovery\ndocument is updated is just the difference between when the respective\ngoroutines finish.\n[same informer]:https://github.com/kubernetes/kubernetes/blob/1a53325550f6d5d3c48b9eecdd123fd84deee879/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go#L192-L210\n\nNote that in HA setup, there is a lag between when apiextension-apiserver\ninstances observe the CRD spec change.\n\n## References\n1. Email thread [kube-apiserver: Self-coordination](https://groups.google.com/d/msg/kubernetes-sig-api-machinery/gTS-rUuEVQY/9bUFVnYvAwAJ)\n"
+  },
+  {
+    "id": "d9b14cc50a4221ed9a7221dfa571d75c",
+    "title": "Insecure Backend Proxy",
+    "authors": ["@deads2k"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["sig-api-machinery", "sig-auth", "sig-cli"],
+    "reviewers": ["@sttts", "@cheftako", "@liggitt", "@soltysh"],
+    "approvers": ["@lavalamp", "@mikedanese"],
+    "editor": "TBD",
+    "creationDate": "2019-09-27",
+    "lastUpdated": "2019-09-27",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Insecure Backend Proxy\n\nWhen trying to get logs for a pod, it is possible for a kubelet to have an expired serving certificate.\nIf a client chooses, it should be possible to bypass the default behavior of the kube-apiserver and allow the kube-apiserver\nto skip TLS verification of the kubelet to allow gathering logs.  This is especially important for debugging\nmisbehaving self-hosted clusters.\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories [optional]](#user-stories-optional)\n    - [Story 1](#story-1)\n    - [Story 2](#story-2)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Drawbacks [optional]](#drawbacks-optional)\n- [Alternatives [optional]](#alternatives-optional)\n- [Infrastructure Needed [optional]](#infrastructure-needed-optional)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n**ACTION REQUIRED:** In order to merge code into a release, there must be an issue in [kubernetes/enhancements] referencing this KEP and targeting a release milestone **before [Enhancement Freeze](https://github.com/kubernetes/sig-release/tree/master/releases)\nof the targeted release**.\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e., [kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be updated for the enhancement to be released.\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP https://github.com/kubernetes/enhancements/issues/1295\n- [x] KEP approvers have set the KEP status to `implementable`\n- [x] Design details are appropriately documented\n- [x] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [x] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nWhen trying to get logs for a pod, it is possible for a kubelet to have an expired serving certificate.\nIf a client chooses, it should be possible to bypass the default behavior of the kube-apiserver and allow the kube-apiserver\nto skip TLS verification of the kubelet to allow gathering logs.  This is safe because the kube-apiserver's credentials\nare always client certificates which cannot be replayed by an evil-kubelet and risk is contained to an evil-kubelet\nreturning false log data.  If the user has chosen to accept this risk, we should allow it for the same reason we\nhave an option for `--insecure-skip-tls-verify`.\n\n## Motivation\n\nOn self-hosted clusters it is possible to end up in a state where a kubelet's serving certificate has expired so a kube-apiserver\ncannot verify the kubelet identity, *but* the kube-apiserver's client certificate is still valid so the kubelet can still\nverify the kube-apiserver.  In this condition, a cluster-admin may need to get pod logs to debug his cluster.\n\n### Goals\n\n1. Allow cluster-admins to get pod logs from kubelets with expired serving certificates.  This will include an API change\nand an addition argument to `kubectl log`\n\n### Non-Goals\n\n1. Allow any bidirectional traffic proxied to kubelets.  This may be a future objective, but is not in scope for the current KEP.\n\n## Proposal\n\nIn [PodLogOptions](https://github.com/kubernetes/api/blob/d58b53da08f5430bb0f4e1154a73314e82b5b3aa/core/v1/types.go), \nadd a `InsecureSkipTLSVerifyBackend bool`\n```go\n// PodLogOptions is the query options for a Pod's logs REST call.\ntype PodLogOptions struct {\n\t// ... existing fields snipped\n\t\n\t// insecureSkipTLSVerifyBackend indicates that the apiserver should not confirm the validity of the \n\t// serving certificate of the backend it is connecting to.  This will make the HTTPS connection between the apiserver\n\t// and the backend insecure. This means the apiserver cannot verify the log data it is receiving came from the real\n\t// kubelet.  If the kubelet is configured to verify the apiserver's TLS credentials, it does not mean the \n\t// connection to the real kubelet is vulnerable to a man in the middle attack (e.g. an attacker could not intercept\n\t// the actual log data coming from the real kubelet).\n\t// +optional\n\tInsecureSkipTLSVerifyBackend bool `json:\"insecureSkipTLSVerifyBackend,omitempty\" protobuf:\"varint,9,opt,name=insecureSkipTLSVerifyBackend\"`\n}\n```\nThe streamer for logs already prevents redirects (see https://github.com/kubernetes/kubernetes/blob/4ee9f007cbc88cca5fa3e8576ff951a52a248e3c/pkg/registry/core/pod/rest/log.go#L83) , so an evil kubelet intercepting this traffic cannot redirect the kube-apiserver\nto use its high powered credentials for a nefarious purpose.\nThe `LocationStreamer` can take an additional argument and the `LogLocation`'s `NodeTransport` can actually produce a purpose\nbuilt transport for insecure connections.\n\nTo make this easier to use, we can add an `--insecure-skip-tls-verify-backend` flag to `kubectl log` which plumbs the option.\nThis part of the KEP is a nice to have, since the kube-apiserver owns the backing capability which has general utility.\n\n### User Stories [optional]\n\n#### Story 1\n\n#### Story 2\n\n### Implementation Details/Notes/Constraints [optional]\n\nThis design is safe based on the following conditions:\n\n1. kube-apiservers only authenticate to kubelets using certificates.  See `--kubelet-client-certificate`.  Certificate based\nauthentication does not send replayable credentials to backends.\n2. Clients must opt-in to the functionality and the documentation must include the impact in plain english.\n3. Evil kubelets cannot trick kube-apiservers into using their credentials for any other purpose.\nIn order to use the kube-apiserver creds, the target of any proxy\nmust terminate the connection.  Since the URL is chosen by the kube-apiserver the evil kubelet cannot rewrite that destination URL.  The destination is the URL for\ngetting logs for one particular pod.\n\n\n### Risks and Mitigations\n\nA super user with write permissions to Node and Pod API objects and read permissions to Pod logs to make the API\nserver exfiltrate data at `https://\u003cnodeName\u003e:\u003cnodePort\u003e/containerLogs/\u003cpodNamespace\u003e/\u003cpodName\u003e/\u003ccontainerName\u003e`, where\nall the bracketed parameters were under their control.  \nThis is ability is already present for someone with full API control via an APIService configured to point to a service\nwith `insecureSkipTLSVerify:true`, with a similar restriction on the path of requests sent to it (must have a leading \n`/apis/\u003cgroup\u003e/\u003cversion\u003e` path prefix).\nThis could affect \nthe only scenarios I can really think of are:\n1. unsecured kubelets from another cluster (or kubelets from another cluster using the same CA)\n2. a non-kubelet endpoint that ignored the specified path and served something confidential (and was unsecured or\nhonored the CA that signed the apiserver's cert)\nBoth scenarios are pretty unlikely.\n\nTrying to restrict the insecureSkipTLSVerify option to ignoring only the expiry date requires reimplementing the TLS \nhandshake with a custom method to \n1. get the serving cert\n2. pull the notAfter to find a time when the cert was valid\n3. construct a set of custom verify options at that time to verify the signature and hostname\nThe risk in doing that is greater than the additional benefit.\n\n\n## Design Details\n\n### Test Plan\n\n1. Positive and negative tests for this are fairly easy to write and the changes are narrow in scope.\n\n### Graduation Criteria\n\nThe problem and solution are well understood and congruent to our past solutions.\nThis new API field will start at beta level.\n\n### Upgrade / Downgrade Strategy\n\nBecause the change is isolated to non-persisted API contracts with the kube-apiserver, there are no skew or upgrade/downgrade considerations.\n\n### Version Skew Strategy\n\nBecause the change is isolated to non-persisted API contracts with the kube-apiserver, there are no skew or upgrade/downgrade considerations.\n\n## Implementation History\n\nMajor milestones in the life cycle of a KEP should be tracked in `Implementation History`.\nMajor milestones might include\n\n- the `Summary` and `Motivation` sections being merged signaling SIG acceptance\n- the `Proposal` section being merged signaling agreement on a proposed design\n- the date implementation started\n- the first Kubernetes release where an initial version of the KEP was available\n- the version of Kubernetes where the KEP graduated to general availability\n- when the KEP was retired or superseded\n\n## Drawbacks [optional]\n\nWhy should this KEP _not_ be implemented.\n\n## Alternatives [optional]\n\nSimilar to the `Drawbacks` section the `Alternatives` section is used to highlight and record other possible approaches to delivering the value proposed by a KEP.\n\n## Infrastructure Needed [optional]\n\nUse this section if you need things from the project/SIG.\nExamples include a new subproject, repos requested, github details.\nListing these here allows a SIG to get the process for these resources started right away.\n"
+  },
+  {
+    "id": "9fda7b5955720384af51d8846ecf4be8",
+    "title": "Enabling clients to tell if resource endpoints serve the same set of objects",
+    "authors": ["@xuchao"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": null,
+    "reviewers": ["@deads2k", "@lavalamp"],
+    "approvers": ["@deads2k", "@lavalamp"],
+    "editor": "",
+    "creationDate": "2018-10-12",
+    "lastUpdated": "2018-12-17",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Enabling clients to tell if resource endpoints serve the same set of objects\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n    - [Correctness](#correctness)\n    - [Efficiency](#efficiency)\n- [Goals](#goals)\n- [Proposal](#proposal)\n  - [API changes to the discovery API](#api-changes-to-the-discovery-api)\n  - [Implementation details](#implementation-details)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nWe propose to expand the discovery API to enable clients to tell if resource\nendpoints (e.g., `extensions/v1beta1/replicaset` and `apps/v1/replicaset`) are\nreferring the same set of objects.\n\n## Motivation\n\nToday, some workload objects (e.g., `replicasets`) are accessible from both the\n`extensions` and the `apps` endpoints. This can happen to more of Kubernetes\nAPIs in the future. Enabling the clients to programmatically detect aliasing\nendpoints can improve both the correctness and the efficiency.\n\n#### Correctness\n* Resource quotas: although `extensions/v1beta1/replicasets` and\n  `apps/v1/replicasets` refer to the same set of objects, they have separate\n  resource quotas today. User can bypass the resource quota of one endpoint by\n  using the other endpoint.\n\n* [Admission webhook rules][]: after a cluster upgrade, if a\n  resource becomes accessible from a new group, the new endpoint will bypass\n  existing rules. For example, here is a webhook configuration that will enforce\n  webhook checks on all replicasets.\n\n  ```yaml\n  rules:\n    - apiGroups:\n      - \"extensions,apps\"\n      apiVersions:\n      - \"*\"\n      resources:\n      - replicasets\n  ```\n\n  However, if in a future release, replicasets are accessible via\n  `fancy-apps/v1` as well, requests sent to `fancy-apps/v1` will be left\n  unchecked. Note that the admins of cluster upgrades are not necessarily the\n  admins of the admission webhooks, so it is not always possible to coordinate\n  cluster upgrades with webhook configuration upgrades.\n\n  If the webhook controller can detect all aliasing resource endpoints, then it\n  can convert `fancy-apps/v1` replicasets to the `apps/v1` and send it to the\n  webhook.\n\n[Admission webhook rules]:https://github.com/kubernetes/kubernetes/blob/18778ea4a151d5f8b346332cb2822b2b0f9d1981/staging/src/k8s.io/api/admissionregistration/v1beta1/types.go#L29\n\n  As a side note, one would expect the [RBAC rules][] would suffer the same\n  problem. However, RBAC is deny by default, so access to the new endpoint is\n  denied unless admin explicitly updates the policy, which is what we want.\n\n[RBAC rules]:https://github.com/kubernetes/kubernetes/blob/18778ea4a151d5f8b346332cb2822b2b0f9d1981/staging/src/k8s.io/api/authorization/v1/types.go#L249\n\n#### Efficiency\n\n* The [storage migrator][] migrates the same objects multiple times if they are\nserved via multiple endpoints.\n\n[storage migrator]:https://github.com/kubernetes-sigs/kube-storage-version-migrator\n\n\n## Goals\n\nThe successful mechanism should\n* enable clients to tell if two resource endpoints refer to the same objects.\n* prevent clients from relying on the implementation details of the mechanism.\n* work for all resources, including built-in resources, custom resources, and\n  aggregated resources.\n\n## Proposal\n\n### API changes to the discovery API\n\nWe add a new field `ResourceID` to the [APIResource][] type. We intentionally avoid\nmentioning any implementation details in the field name or in the comment.\n\n[APIResource]:https://github.com/kubernetes/kubernetes/blob/f22334f14d92565ec3ff9d4ff2b995eae9af622a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go#L881-L905\n\n```golang\ntype APIResource struct {\n        // An opaque token which can be used to tell if separate resources\n        // refer to the same underlying objects. This allows clients to follow\n        // resources that are served at multiple versions and/or groups.\n        ResourceID string\n        // These are the existing fields.\n        Name string\n        SingularName string\n        Namespaced bool\n        Group string\n        Version string\n        Kind string\n        Verbs Verbs\n        ShortNames []string\n        Categories []string\n}\n```\n\n### Implementation details\n\nFor built-in resources, their `resourceID`s are set to `SHA256(\u003cetcd key\nprefix\u003e)`. For example, for both `extensions/v1beta1/replicasets` and\n`apps/v1/replicasets`, the etcd key prefix is `/registry/replicasets`, so their\nresourceIDs are the same. Serving the hashed prefix instead of the prefix in\nplain text is to encourage the clients to only test the equality of the hashed\nvalues, instead of relying on the absolute value.\n\nFor custom resources, their `resourceID`s are also set to `SHA256(\u003cetcd key\nprefix\u003e)`. In the current implementation, the etcd key prefix for a custom\nresource is `/registry/\u003ccrd.spec.group\u003e/\u003ccrd.spec.names.plural\u003e`.\n\nFor aggregated resources, because their discovery doc is fully controlled\nby the aggregated apiserver, the kube-apiserver has no means to validate their\n`resourceID`. If the server is implemented with the generic apiserver library,\nthe `resourceID` will be `SHA256(\u003cetcd key prefix\u003e)`.\n\nFor subresources, the `resourceID` field is left empty for simplicity. Clients\ncan tell if two subresource endpoints refer the same objects by checking if the\n`resourceID`s of the main resources are the same.\n\nFor non-persistent resources like `tokenReviews` or `subjectAccessReviews`,\nthough the objects are not persisted, the [forward compatibility][] motivation\nstill applies, e.g., admins might configure the admission webhooks to intercept\nrequests sent to all endpoints. Thus, the `resourceID` cannot be left empty, it\nwill be set to `SHA256(\u003cthe would-be etcd key prefix\u003e)`, e.g., for\n`tokenReviews`, it's `SHA256(/registry/tokenreviews)`. \n\n[forward compatibility]:#broken-forwards-compatibility\n\n### Risks and Mitigations\n\nIn the future, the \"etcd key prefix\" might not be sufficient to uniquely\nidentify a set of objects. We can always add more factors to the `resourceID` to\nensure their uniqueness. It does not break backwards compatibility because the\n`resourceID` is opaque.\n\nAnother risk is that an aggregated apiserver accidentally reports `resourceID`\nthat's identical to the built-in or the custom resources, this will confuse\nclients. Because the kube-apiserver has zero control over the discovery doc of\naggregated resources, it cannot do any validation to prevent this kind of error.\nIt will be aggregated apiserver provider's responsibility to prevent such errors.\n\n## Graduation Criteria\n\nThe field will be alpha in 1.14, protected by a feature flag. By default the\n`ResourceID` is not shown in the discovery document as the feature flag\nis default to `False`.\n\nIf we don't find any problem with the field, we will promote it to beta in 1.15\nand GA in 1.16. Otherwise we just remove the field while keeping a tombstone\nfor the protobuf tag.\n\nThe above is a simplified version of the Kubernetes API change [guideline][],\nbecause the discovery API is read-only.\n\n[guideline]:https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api_changes.md#alpha-field-in-existing-api-version\n\n## Alternatives\n1. Adding to the discovery API a reference to the canonical endpoint. For\n   example, in the discovery API, `extensions/v1beta1/replicasets` reports\n   `apps/v1/replicasets` as the canonical endpoint. This approach is similar to\n   `resourceID` proposal, but because the resource names are explicitly exposed,\n   clients might use the information in unintended ways.\n\n2. Serving a list of all sets of aliasing resources via a new API. Aggregated\n   apiservers make such a design complex. For example, we will need to design how\n   the aggregated apiserver registers its resource aliases. \n\n3. Hard coding UUIDs for built-in resources, instead of hashes. This doesn't\n   work for CRDs.\n"
+  },
+  {
+    "id": "d30ea04876b6ce6dc068f4037a55deb5",
+    "title": "exposing hashed storage versions via the discovery API",
+    "authors": ["@xuchao"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": null,
+    "reviewers": ["@deads2k", "@lavalamp"],
+    "approvers": ["@deads2k", "@lavalamp"],
+    "editor": "",
+    "creationDate": "2019-01-02",
+    "lastUpdated": "2019-01-04",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Exposing storage versions in opaque values via the discovery API\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Terms](#terms)\n- [Summary](#summary)\n- [Motivation](#motivation)\n- [Proposal](#proposal)\n  - [API changes to the discovery API](#api-changes-to-the-discovery-api)\n  - [Implementation details](#implementation-details)\n- [Graduation Criteria](#graduation-criteria)\n- [Risks and mitigation](#risks-and-mitigation)\n  - [HA masters](#ha-masters)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Terms\n\n**storage versions**: Kubernetes API objects are converted to specific API\nversions before stored in etcd. \"Storage versions\" refers to these API versions.\n\n## Summary\n\nWe propose to expose the hashed storage versions in the discovery API.\n\n## Motivation\n\nWe intend to use the exposed storage version hash to trigger the [storage\nversion migrator][].\n\nIn short, the storage version migrator detects if objects in etcd are stored in\na version different than the configured storage version. If so, the migrator\nissues no-op update for the objects to migrate them to the storage version.\n\nThe storage version migrator can keep track of the versions the objects are\nstored as. However, today the migrator has no way to tell what the expected\nstorage versions are. Thus we propose to expose this piece of information via\nthe discovery API.\n\n[storage version migrator]:https://github.com/kubernetes-sigs/kube-storage-version-migrator\n\n## Proposal\n\n### API changes to the discovery API\n\nWe add a new field `StorageVersionHash` to the [APIResource][] type.\n\nSo far, the only valid consumer of this information is the storage version\nmigrator, which only needs to do equality comparisons on the storage versions.\nHence, we use the hash value instead of the plain text to avoid clients misusing\nthe information. The hash function needs to ensure hash value differs if the\nstorage versions are different.\n\n[APIResource]:https://github.com/kubernetes/kubernetes/blob/f22334f14d92565ec3ff9d4ff2b995eae9af622a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go#L881-L905\n\n```golang\ntype APIResource struct {\n        // The hash value of the storage version, the version this resource is\n        // converted to when written to the data store. Value must be treated \n        // as opaque by clients. Only equality comparison on the value is valid.\n\t// This is an alpha feature and may change or be removed in the future.\n        // The field is populated by the apiserver only if the\n        // StorageVersionHash feature gate is enabled.\n        // This field will remain optional even if it graduates. \n        // +optional\n        StorageVersionHash string `json:\"storageVersionHash,omitempty\"`\n        // These are the existing fields.\n        Name string\n        SingularName string\n        Namespaced bool\n        Group string\n        Version string\n        Kind string\n        Verbs Verbs\n        ShortNames []string\n        Categories []string\n}\n```\n\n### Implementation details\n\nThe hash function needs to ensure hash value differs if the storage versions are\ndifferent. SHA-256 is good enough for our purpose.\n\nFor different categories of resources,\n\n* For built-in resources, the kube-apiserver will set the `StorageVersionHash`\nwhen it bootstraps.\n\n* For custom resources, the storage version is already exposed through the [CRD\nspec][]. The apiextension apiserver will hash the value and sets the\n`StorageVersionHash` when registers the discovery doc for the CRD.\n\n* For aggregated resources, if the aggregated apiserver is implemented using the\ngeneric apiserver library, the `StorageVersionHash` will be set in the same way\nas the kube-apiserver. Otherwise, the aggregated apiserver is responsible\nto come up with a proper `StorageVersionHash`.\n\n* For sub-resources, the `StorageVersionHash` field will be left empty. No\nsub-resource persists in the data store and thus does not require storage\nversion migration.\n\n[CRD spec]:https://github.com/kubernetes/kubernetes/blob/7d8554643e2e05fda714f30fc71f34ce05514b68/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go#L167\n\n## Graduation Criteria\n\nThe discovery API is read-only and the `StorageVersionHash` field is only\nintended to be used by the storage version migrator, the graduation story is\nsimple.\n\nThe field will be alpha in 1.14, protected by a feature flag. By default the\n`StorageVersionHash` is not shown in the discovery document as the feature flag\nis default to `False`.\n\nIf we don't find any problem with the field, we will promote it to beta in 1.15\nand GA in 1.16. Otherwise we just remove the field while keeping a tombstone\nfor the protobuf tag.\n\nThe above is a simplified version of Kubernetes API change [guideline][].\n\n[guideline]:https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api_changes.md#alpha-field-in-existing-api-version\n\n## Risks and mitigation\n\n### HA masters\n\nKubernetes does not have a convergence mechanism for HA masters. During HA\nmaster rolling upgrade/downgrade, depending on which apiserver handles the\nrequest, the discovery document and the storage version may vary.\n\nThis breaks the auto-triggered storage migration. For example, the storage\nversion migrator gets the discovery document from an upgraded apiserver. The new\nstorage version for *deployment* is `apps/v1`, while the old storage version is\n`apps/v1beta2`. The migrator issues no-op updates for all `deployments` to\nmigrate them to `apps/v1`. However, other clients might write to `deployments`\nvia the other to-be-upgraded apiservers and thus accidentally revert the\nmigration. The migrator cannot detect the reversion.\n\nIdeally, the HA masters can detect that the masters haven't converged on the\nstorage versions, and manifest the disagreement in the discovery API. The\nmigrator waits for the masters to converge before starting migration. Before\nsuch a convergence mechanism exists, the auto-triggered storage version\nmigration is **not** safe for HA masters.\n\nTo workaround, the cluster admins of HA clusters need to\n* turn off the migration triggering controller, which is a standalone controller.\n* manually trigger migration after the rolling upgrade/downgrade is done.\n\nThough the `StorageVersionHash` cannot be used to automate HA cluster storage\nmigration, manually triggered migration can determine use this information if\nthe storage version for a particular resource has changed since the last run and\nskip unnecessary migrations.\n\n## Alternatives\n1. We had considered triggering the storage version migrator by the change of\n   the apiserver's binary version. However, the correctness of this approach\n   relies on that the storage versions never change without changing the binary\n   version first. This might not hold true in the future.\n"
+  },
+  {
+    "id": "722352bce343f1fd26fc98572fe02da3",
+    "title": "Automated Storage Version Migration with Storage Version Hash",
+    "authors": ["@xuchao"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": null,
+    "reviewers": ["@deads2k", "@yliaog"],
+    "approvers": ["@deads2k", "@lavalamp"],
+    "editor": "",
+    "creationDate": "2019-01-23",
+    "lastUpdated": "2019-01-23",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Goal](#goal)\n- [API design](#api-design)\n- [Storage migration triggering controller](#storage-migration-triggering-controller)\n- [Implications to cluster operators](#implications-to-cluster-operators)\n- [Life-cycle of a StorageState object](#life-cycle-of-a-storagestate-object)\n- [Future work: HA clusters](#future-work-ha-clusters)\n- [Future work: persisted discovery document](#future-work-persisted-discovery-document)\n\u003c!-- /toc --\u003e\n\n## Goal\n\nAs the discovery document now exposes the [storage version hash][], storage version\nmigration should start automatically when the hash changes.\n\n[storage version hash]:https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/35-storage-version-hash.md\n\n## API design\n\nWe introduce the `StorageState` API, as a CRD defined in the `migration.k8s.io`\ngroup.\n\n```golang\n// StorageState is the state of the storage, for a specific resource.\ntype StorageState struct {\n  metav1.TypeMeta\n  // The name is \"\u003cresource\u003e.\u003cgroup\u003e\". The API validation will enforce it by\n  // comapring the name with the spec.resource.\n  metav1.ObjectMeta\n  Spec StorageStateSpec\n  Status StorageStateStatus\n}\n\ntype StorageStateSpec {\n  // The resource this StorageState is about.\n  Resource GroupResource\n}\n\ntype StorageStateStatus {\n  // The hash values of storage versions that persisted instances of\n  // spec.resource might still be encoded in.\n  // \"Unknown\" is a valid value in the list, and is the default value.\n  // It is not safe to upgrade or downgrade to an apiserver binary that does not\n  // support all versions listed in this field, or if \"Unknown\" is listed.\n  // Once the storage version migration for this resource has completed, the\n  // value of this field is refined to only contain the\n  // currentStorageVersionHash.\n  // Once the apiserver has changed the storage version, the new storage version\n  // is appended to the list.\n  // +optional\n  PersistedStorageVersionHashes []string\n  // The hash value of the current storage version, as shown in the discovery\n  // document served by the API server.\n  // Storage Version is the version to which objects are converted to\n  // before persisted.\n  CurrentStorageVersionHash string\n  // LastHeartbeatTime is the last time the storage migration triggering\n  // controller checks the storage version hash of this resource in the\n  // discovery document and updates this field.\n  // +optional\n  LastHeartbeatTime metav1.Time\n}\n``` \n\nWe had considered making `PersistedStorageVersionHashes` part of the `spec`\ninstead of the `status`, because the stored versions cannot be deduced\nimmediately by inspecting Kubernetes API. However, we decided to put it in the\nstatus because\n* Its value eventually is determined by the `StorageVersionMigration` API, i.e.,\n  it's set to the storage version when the corresponding migration has\n  completed.\n* Even though we cannot reconstruct this field immediately, this field is\n  initialized to [\"Unknown\"], which truthfully reflects the state.\n* Putting it in the spec is even more unnatural. Spec describes the desired\n  state the system should eventually reach in the future, while this field is\n  about states in the history.\n* Keeping it in the status is consistent with the [CRD.status.StoredVersions][]\n  API.\n\n[CRD.status.StoredVersions]:https://github.com/kubernetes/kubernetes/blob/697c2316faaabae8ef8371032b60be65d7795e68/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go#L305\n  \n## Storage migration triggering controller\n\nWe will add a controller that monitors the discovery documents and the\nstorageStates to trigger storage version migration.\n\nThe controller fetches the discovery document periodically. For each resource,\nif the storageState doesn't exist for the resource, the controller\n  * stops any ongoing migration of this resource by deleting existing\n    [storageVersionMigrations][],\n  * creates a new storageVersionMigration for this resource,\n  * creates the storageState for the resource, setting\n    .status.persistedStorageVersionHashes to [\"Unknown\"], and setting\n    status.currentStorageVersionHash to the value shown in the discovery\n    document.\n\nIf the storageState exists, the controller compares the storageVersionHash in\nthe discovery document with storageState.status.currentStorageVersionHash. If\nthey are the same, the controller simply updates the status.lastHeartbeatTime\nand is done.\nOtherwise, the controller\n  * stops any ongoing migration of this resource by deleting existing\n    [storageVersionMigrations][],\n  * creates a new storageVersionMigration for this resource,\n  * as a single operation, updates the status.lastHeartbeatTime, sets\n    storageState.status.currentStorageVersionHash to the value shown in the\n    discovery document, and appends the hash to the\n    storageState.status.persistedStorageVersionHashes.\n\nThe refresh rate of the discovery document needs to satisfy two conflicting\ngoals: *i)* not causing too much traffic to apiservers, and *ii)* not missing\nstorage version configuration changes. More precisely, the storage version of a\ncertain resource should not change twice within a refresh period. 10 minutes is\na good middle ground.\n\nThe controller uses an informer to monitor all [storageVersionMigrations][]. If a\nmigration completes, the controller finds the storageState of that resource,\nupdates its status.persistedStorageVersionHashes to equal its\nstatus.currentStorageVersionHash.\n\nIf the controller crash loops and misses storage version changes, the\nstorageState.status.persistedStorageVersionHashes can be stale. Thus, when the\ncontroller bootstraps, for every storageState object, it checks the\n.status.lastHeartbeatTime, if the timestamp is more than 10 minutes old, it\nrecreates storageState object with persistedStorageVersionHashes set to [\"Unknown\"].\n\n[storageVersionMigrations]:https://github.com/kubernetes-sigs/kube-storage-version-migrator/blob/444c1beafd4a22684c2b4ba50fa489ec24873c10/pkg/apis/migration/v1alpha1/types.go#L29\n\n## Implications to cluster operators\n\n* Cluster operators can do fast upgrade/downgrade, without waiting for all\n  storage migrations to complete, which had been a requirement in the [alpha\n  workflow][]. Instead, the cluster operators just need to make sure the\n  to-be-deployed apiserver binaries understand all versions recorded in\n  storageState.status.persistedStorageVersionHashes. This is useful when a\n  cluster needs to roll back.\n\n* On the other hand, if the cluster operators consecutively upgrade or downgrade\n  the cluster in a 10 minute window, the migration triggering controller, which\n  polls the discovery doc every 10 minutes, will miss the intermediate states of\n  the cluster. The storageState will be inaccurate, which will endanger\n  the next upgrade/downgrade. Operators need to manually delete all the\n  storageState objects to reset.\n\n* The caveat applies to Custom Resource admins, too. If the storage version of a\n  CR changes twice in a 10 minute window, admin needs to manually delete the\n  storageState of the CR to reset.\n\n[alpha workflow]:https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/0030-storage-migration.md#alpha-workflow\n\n## Life-cycle of a StorageState object\n\nLet's put all the bits together and describe the life cycle\nof a StorageState object. We will use the StorageState of CronJob as an\nexample.\n\nWhen the migration triggering controller is installed on an 1.14 cluster for the\nfirst time, there is no StorageState object. The controller creates the\nStorageState object for CronJob, setting .status.persistedStorageVersionHashes to\n[\"Unknown\"] and .status.currentStorageVersionHash to batch/v2alpha1. It also\ncreates a storageVersionMigration object to request a migration. When the\nmigration completed, the migration triggering controller sets\n.status.persistedStorageVersionHashes to [batch/v2alpha1].\n\nThe migration triggering controller crashes and has been offline for less than\n10 minutes. When it's back online, it sees the .status.lastHeartbeatTime is less\nthan 10 minutes old, thus it considers the storageState still valid. The\ncontroller does nothing other than updating the lastHeartbeatTime.\n\nThe cluster admin upgrades the cluster to 1.15. Let's assume the storage version\nof the cronjob is changed to batch/v2beta1. The migration triggering controller\nobserves the change. It sets the storageState.status.currentStorageVersionHash to\nbatch/v2beta1 and storageState.status.persistedStorageVersionHashes to [v2alpha1,\nv2beta1].  It also creates a migration request for cronjobs to migrate to\nv2beta1.\n\nThe cluster admin finds a critical issue in 1.15 and downgrades the cluster\nback to 1.14. The migration triggering controller sets the\nstorageState.status.currentStorageVersionHash to batch/v2alpha1 and recreates the\nstorageVersionMigration object to migrate to v2alpha1.\n\n## Future work: HA clusters\n\nWhen the masters of an HA cluster undergoes rolling upgrade, the storage\nversions might be configured differently in different API servers. Storage\nmigration shouldn't start until the rolling upgrade is done.\n\nIdeally, for an HA cluster undergoing rolling upgrade, the following should\nhappen for each resource: \n* the discovery document should expose the lack of consensus by listing all\n  storage version hashes supported by different API servers.\n* the storage migration triggering controller should\n  * add all listed storage version hashes to\n    storageState.status.persistedStorageVersionHashes.\n  * set storageState.status.currentStorageVersionHash to `Unknown`.\n  * delete any in progress migration by deleting existing\n    storageVersionMigrations.\n\nAfter the API server rolling upgrade is done, the discovery document would\nexpose the agreed storage version hashes, and the storage migration triggering\ncontroller will resume the work described in the [previous section][].\n\n[previous section]:storage-migration-triggering-controller\n\nHow to develop a convergence mechanism for HA masters is beyond the scope of\nthis KEP. We will implement this KEP for non-HA case first. Operators of HA\nclusters need to disable the triggering controller and start migration\nmanually, e.g., deploying the [initializer][] manually.\n\n[initializer]:https://github.com/kubernetes-sigs/kube-storage-version-migrator/tree/master/cmd/initializer\n\n## Future work: persisted discovery document\n\nToday, the discovery document is held in apiserver's memory. Previous state is\nlost when the apiserver reboots. Thus we need the migration triggering\ncontroller to poll the apiserver and keep track of the stored versions. And thus\nwe have the limit that the storage version cannot change more than once in\nthe controller's polling period. If the apiserver persists the discovery\ndocument in etcd, then we can remove the limit.\nPersisting the discovery document in etcd also helps HA apiservers to agree on\nthe discovery document.\n"
+  },
+  {
+    "id": "97cc4949f4362533a663d018d1399b10",
+    "title": "TTL After Finished",
+    "authors": ["@janetkuo"],
+    "owningSig": "sig-apps",
+    "participatingSigs": ["sig-api-machinery"],
+    "reviewers": ["@enisoc", "@tnozicka"],
+    "approvers": ["@kow3ns"],
+    "editor": "TBD",
+    "creationDate": "2018-08-16",
+    "lastUpdated": "2018-08-16",
+    "status": "provisional",
+    "seeAlso": ["n/a"],
+    "replaces": ["n/a"],
+    "supersededBy": ["n/a"],
+    "markdown": "\n# TTL After Finished Controller\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n- [Proposal](#proposal)\n  - [Concrete Use Cases](#concrete-use-cases)\n  - [Detailed Design](#detailed-design)\n    - [Feature Gate](#feature-gate)\n    - [API Object](#api-object)\n      - [Validation](#validation)\n  - [User Stories](#user-stories)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n    - [TTL Controller](#ttl-controller)\n    - [Finished Jobs](#finished-jobs)\n    - [Finished Pods](#finished-pods)\n    - [Owner References](#owner-references)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nWe propose a TTL mechanism to limit the lifetime of finished resource objects,\nincluding Jobs and Pods, to make it easy for users to clean up old Jobs/Pods\nafter they finish. The TTL timer starts when the Job/Pod finishes, and the\nfinished Job/Pod will be cleaned up after the TTL expires.\n\n## Motivation\n\nIn Kubernetes, finishable resources, such as Jobs and Pods, are often\nfrequently-created and short-lived. If a Job or Pod isn't controlled by a\nhigher-level resource (e.g. CronJob for Jobs or Job for Pods), or owned by some\nother resources, it's difficult for the users to clean them up automatically,\nand those Jobs and Pods can accumulate and overload a Kubernetes cluster very\neasily. Even if we can avoid the overload issue by implementing a cluster-wide\n(global) resource quota, users won't be able to create new resources without\ncleaning up old ones first. See [#64470][].\n\nThe design of this proposal can be later generalized to other finishable\nfrequently-created, short-lived resources, such as completed Pods or finished\ncustom resources.\n\n[#64470]: https://github.com/kubernetes/kubernetes/issues/64470\n\n### Goals\n\nMake it easy to for the users to specify a time-based clean up mechanism for\nfinished resource objects. \n* It's configurable at resource creation time and after the resource is created.\n\n## Proposal\n\n[K8s Proposal: TTL controller for finished Jobs and Pods][]\n\n[K8s Proposal: TTL controller for finished Jobs and Pods]: https://docs.google.com/document/d/1U6h1DrRJNuQlL2_FYY_FdkQhgtTRn1kEylEOHRoESTc/edit\n\n### Concrete Use Cases\n\n* [Kubeflow][] needs to clean up old finished Jobs (K8s Jobs, TF Jobs, Argo\n  workflows, etc.), see [#718][].\n\n* [Prow][] needs to clean up old completed Pods \u0026 finished Jobs. Currently implemented with Prow sinker.\n\n* [Apache Spark on Kubernetes][] needs proper cleanup of terminated Spark executor Pods.\n\n* Jenkins Kubernetes plugin creates slave pods that execute builds. It needs a better way to clean up old completed Pods.\n\n[Kubeflow]: https://github.com/kubeflow\n[#718]: https://github.com/kubeflow/tf-operator/issues/718\n[Prow]: https://github.com/kubernetes/test-infra/tree/master/prow\n[Apache Spark on Kubernetes]: http://spark.apache.org/docs/latest/running-on-kubernetes.html\n\n### Detailed Design \n\n#### Feature Gate\n\nThis will be launched as an alpha feature first, with feature gate\n`TTLAfterFinished`.\n\n#### API Object\n\nWe will add the following API fields to `JobSpec` (`Job`'s `.spec`).\n\n```go\ntype JobSpec struct {\n \t// ttlSecondsAfterFinished limits the lifetime of a Job that has finished\n\t// execution (either Complete or Failed). If this field is set, once the Job\n\t// finishes, it will be deleted after ttlSecondsAfterFinished expires. When\n\t// the Job is being deleted, its lifecycle guarantees (e.g. finalizers) will\n\t// be honored. If this field is unset, ttlSecondsAfterFinished will not\n\t// expire. If this field is set to zero, ttlSecondsAfterFinished expires\n\t// immediately after the Job finishes.\n\t// This field is alpha-level and is only honored by servers that enable the\n\t// TTLAfterFinished feature.\n\t// +optional\n\tTTLSecondsAfterFinished *int32\n}\n```\n\nThis allows Jobs to be cleaned up after they finish and provides time for\nasynchronous clients to observe Jobs' final states before they are deleted.\n\n\nSimilarly, we will add the following API fields to `PodSpec` (`Pod`'s `.spec`).\n\n```go\ntype PodSpec struct {\n \t// ttlSecondsAfterFinished limits the lifetime of a Pod that has finished\n\t// execution (either Succeeded or Failed). If this field is set, once the Pod\n\t// finishes, it will be deleted after ttlSecondsAfterFinished expires. When\n\t// the Pod is being deleted, its lifecycle guarantees (e.g. finalizers) will\n\t// be honored. If this field is unset, ttlSecondsAfterFinished will not\n\t// expire. If this field is set to zero, ttlSecondsAfterFinished expires\n\t// immediately after the Pod finishes.\n\t// This field is alpha-level and is only honored by servers that enable the\n\t// TTLAfterFinished feature.\n\t// +optional\n\tTTLSecondsAfterFinished *int32\n}\n```\n\n##### Validation\n\nBecause Job controller depends on Pods to exist to work correctly. In Job\nvalidation, `ttlSecondsAfterFinished` of its pod template shouldn't be set, to\nprevent users from breaking their Jobs. Users should set TTL seconds on a Job,\ninstead of Pods owned by a Job.\n\nIt is common for higher level resources to call generic PodSpec validation;\ntherefore, in PodSpec validation, `ttlSecondsAfterFinished` is only allowed to\nbe set on a PodSpec with a `restartPolicy` that is either `OnFailure` or `Never`\n(i.e. not `Always`).\n\n### User Stories\n\nThe users keep creating Jobs in a small Kubernetes cluster with 4 nodes.\nThe Jobs accumulates over time, and 1 year later, the cluster ended up with more\nthan 100k old Jobs. This caused etcd hiccups, long high latency etcd requests,\nand eventually made the cluster unavailable.\n\nThe problem could have been avoided easily with TTL controller for Jobs.\n\nThe steps are as easy as:\n\n1. When creating Jobs, the user sets Jobs' `.spec.ttlSecondsAfterFinished` to\n   3600 (i.e. 1 hour).\n1. The user deploys Jobs as usual.\n1. After a Job finishes, the result is observed asynchronously within an hour\n   and stored elsewhere.\n1. The TTL collector cleans up Jobs 1 hour after they complete.\n\n### Implementation Details/Notes/Constraints\n\n#### TTL Controller\nWe will add a TTL controller for finished Jobs and finished Pods. We considered\nadding it in Job controller, but decided not to, for the following reasons:\n\n1. Job controller should focus on managing Pods based on the Job's spec and pod\n   template, but not cleaning up Jobs.\n1. We also need the TTL controller to clean up finished Pods, and we consider\n   generalizing TTL controller later for custom resources. \n\nThe TTL controller utilizes informer framework, watches all Jobs and Pods, and\nread Jobs and Pods from a local cache.\n\n#### Finished Jobs\n\nWhen a Job is created or updated:\n\n1. Check its `.status.conditions` to see if it has finished (`Complete` or\n   `Failed`). If it hasn't finished, do nothing. \n1. Otherwise, if the Job has finished, check if Job's \n   `.spec.ttlSecondsAfterFinished` field is set. Do nothing if the TTL field is\n   not set. \n1. Otherwise, if the TTL field is set, check if the TTL has expired, i.e. \n   `.spec.ttlSecondsAfterFinished` + the time when the Job finishes\n   (`.status.conditions.lastTransitionTime`) \u003e now. \n1. If the TTL hasn't expired, delay re-enqueuing the Job after a computed amount\n   of time when it will expire. The computed time period is:\n   (`.spec.ttlSecondsAfterFinished` + `.status.conditions.lastTransitionTime` -\n   now).\n1. If the TTL has expired, `GET` the Job from API server to do final sanity\n   checks before deleting it.\n1. Check if the freshly got Job's TTL has expired. This field may be updated\n   before TTL controller observes the new value in its local cache.\n   * If it hasn't expired, it is not safe to delete the Job. Delay re-enqueue\n     the Job after a computed amount of time when it will expire.\n1. Delete the Job if passing the sanity checks. \n\n#### Finished Pods\n\nWhen a Pod is created or updated:\n1. Check its `.status.phase` to see if it has finished (`Succeeded` or `Failed`).\n   If it hasn't finished, do nothing. \n1. Otherwise, if the Pod has finished, check if Pod's\n   `.spec.ttlSecondsAfterFinished` field is set. Do nothing if the TTL field is\n   not set. \n1. Otherwise, if the TTL field is set, check if the TTL has expired, i.e.\n   `.spec.ttlSecondsAfterFinished` + the time when the Pod finishes (max of all\n   of its containers termination time\n   `.containerStatuses.state.terminated.finishedAt`) \u003e now. \n1. If the TTL hasn't expired, delay re-enqueuing the Pod after a computed amount\n   of time when it will expire. The computed time period is:\n   (`.spec.ttlSecondsAfterFinished` + the time when the Pod finishes - now).\n1. If the TTL has expired, `GET` the Pod from API server to do final sanity\n   checks before deleting it.\n1. Check if the freshly got Pod's TTL has expired. This field may be updated\n   before TTL controller observes the new value in its local cache.\n   * If it hasn't expired, it is not safe to delete the Pod. Delay re-enqueue\n     the Pod after a computed amount of time when it will expire.\n1. Delete the Pod if passing the sanity checks. \n\n#### Owner References\n\nWe have considered making TTL controller leave a Job/Pod around even after its\nTTL expires, if the Job/Pod has any owner specified in its\n`.metadata.ownerReferences`.\n\nWe decided not to block deletion on owners, because the purpose of\n`.metadata.ownerReferences` is for cascading deletion, but not for keeping an\nowner's dependents alive. If the Job is owned by a CronJob, the Job can be\ncleaned up based on CronJob's history limit (i.e. the number of dependent Jobs\nto keep), or CronJob can choose not to set history limit but set the TTL of its\nJob template to clean up Jobs after TTL expires instead of based on the history\nlimit capacity. \n\nTherefore, a Job/Pod can be deleted after its TTL expires, even if it still has\nowners. \n\nSimilarly, the TTL won't block deletion from generic garbage collector. This\nmeans that when a Job's or Pod's owners are gone, generic garbage collector will\ndelete it, even if it hasn't finished or its TTL hasn't expired. \n\n### Risks and Mitigations\n\nRisks:\n* Time skew may cause TTL controller to clean up resource objects at the wrong\n  time.\n\nMitigations:\n* In Kubernetes, it's required to run NTP on all nodes ([#6159][]) to avoid time\n  skew. We will also document this risk.\n\n[#6159]: https://github.com/kubernetes/kubernetes/issues/6159#issuecomment-93844058\n\n## Graduation Criteria\n\nWe want to implement this feature for Pods/Jobs first to gather feedback, and\ndecide whether to generalize it to custom resources. This feature can be\npromoted to beta after we finalize the decision for whether to generalize it or\nnot, and when it satisfies users' need for cleaning up finished resource\nobjects, without regressions.\n\nThis will be promoted to GA once it's gone a sufficient amount of time as beta\nwith no changes. \n\n[umbrella issues]: https://github.com/kubernetes/kubernetes/issues/42752\n\n## Implementation History\n\nTBD\n"
+  },
+  {
+    "id": "50f68c429ed17bb419aa1485a07bd30e",
+    "title": "Optional Service Environment Variables",
+    "authors": ["@bradhoekstra", "@kongslund"],
+    "owningSig": "sig-apps",
+    "participatingSigs": null,
+    "reviewers": ["TBD"],
+    "approvers": ["TBD"],
+    "editor": "TBD",
+    "creationDate": "2018-09-25",
+    "lastUpdated": "2018-09-25",
+    "status": "provisional",
+    "seeAlso": ["https://github.com/kubernetes/community/pull/1249"],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Optional Service Environment Variables\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis enhancement allows application developers to choose whether their Pods will receive [environment variables](https://kubernetes.io/docs/concepts/services-networking/service/#environment-variables) from services in their namespace. They can choose to disable them via the new `enableServiceLinks` field in `PodSpec`. The current behaviour will continue to be the default behaviour, but the developer may choose to disable these environment variables for certain workloads for reasons such as incompatibilities with other expected environment variables or scalability issues.\n\n## Motivation\n\nToday, a list of all services that were running when a pod's containers are created is automatically injected to those containers as environment variables matching the syntax of Docker links. There is no way to disable this.\n\nDocker links have long been considered as a [deprecated legacy feature](https://docs.docker.com/engine/userguide/networking/default_network/dockerlinks/) of Docker since the introduction of networks and DNS. Likewise, in Kubernetes, DNS is to be preferred over service links.\n\nPossible issues with injected service links are:\n\n* Accidental coupling.\n* Incompatibilities with container images that no longer utilize service links and explicitly fail at startup time if certain service links are defined.\n* Performance penalty in starting up pods [for namespaces with many services](https://github.com/kubernetes/kubernetes/issues/1768#issuecomment-330778184)\n\n### Goals\n\n* Allow users to choose whether to inject service environment variables in their Pods.\n* Do this in a backwards-compatible, non-breaking way. Default to the current behaviour.\n\n### Non-Goals\n\nN/A\n\n## Proposal\n\n### User Stories\n\n* As an application developer, I want to be able to disable service link injection since the injected environment variables interfere with a Docker image that I am trying to run on Kubernetes.\n* As an application developer, I want to be able to disable service link injection since I don't need it and it takes increasingly longer time to start pods as services are added to the namespace.\n* As an application developer, I want to be able to disable service link injection since pods can fail to start if the environment variable list becomes too long. This can happen when there are \u003e5,000 services in the same namespace.\n\n### Implementation Details/Notes/Constraints\n\n`PodSpec` is extended with an additional field, `enableServiceLinks`. The field should be a pointer to a boolean and default to true if nil.\n\nIn `kubelet_pods.go`, the value of that field is passed along to the function `getServiceEnvVarMap` where it is used to decide which services will be propogated into environment variables. In case `enableServiceLinks` is false then only the `kubernetes` service in the `kl.masterServiceNamespace` should be injected. The latter is needed in order to preserve Kubernetes variables such as `KUBERNETES_SERVICE_HOST` since a lot of code depends on it.\n\n### Risks and Mitigations\n\nThe current behaviour is being kept as the default as much existing code and documentation depends on these environment variables.\n\n## Graduation Criteria\n\nN/A\n\n## Implementation History\n\n- 2017-10-21: First draft of original proposal [PR](https://github.com/kubernetes/community/pull/1249)\n- 2018-02-22: First draft of implementation [PR](https://github.com/kubernetes/kubernetes/pull/60206)\n- 2018-08-31: General consensus of implementation plan\n- 2018-09-17: First draft of new implementation [PR](https://github.com/kubernetes/kubernetes/pull/68754)\n- 2018-09-24: Implementation merged into master\n- 2018-09-25: Converting proposal into this KEP\n"
+  },
+  {
+    "id": "95eb95cbbf9d4f528a843b6b61eedadc",
+    "title": "Portable Service Definitions",
+    "authors": ["@mattfarina"],
+    "owningSig": "sig-apps",
+    "participatingSigs": ["sig-service-catalog"],
+    "reviewers": ["@carolynvs", "@kibbles-n-bytes", "@duglin", "@jboyd01", "@prydonius", "@kow3ns"],
+    "approvers": ["@mattfarina", "@prydonius", "@kow3ns"],
+    "editor": "TBD",
+    "creationDate": "2018-11-13",
+    "lastUpdated": "2018-11-19",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "# KEP: Portable Service Definitions\n\n\n# Portable Service Definitions\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Story 1](#story-1)\n    - [Story 2](#story-2)\n    - [Story 3](#story-3)\n    - [Story 4](#story-4)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Drawbacks [optional]](#drawbacks-optional)\n- [Alternatives [optional]](#alternatives-optional)\n- [Infrastructure Needed [optional]](#infrastructure-needed-optional)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThe goal of this feature is to enable an application to be deployed into multiple environments while relying on external services that are not part of the application and using the same objects in all environments. That includes service instances that may or may not be managed by Kubernetes. For example, take a WordPress application that relies on MySQL. If the application is running in GKE or AKS it may leverage a Google or Azure MySQL as a Service. If it is running in Minikube or in a bare metal cluster it may use MySQL managed by the Kubernetes cluster. In all of these cases, the same resource is declared asking for the service (e.g., MySQL) and the credentials to use the service are returned in a common manner (e.g., a secret with the same type and schema). This enables portability from one environment to another because working with services in all environments leverages the same Kubernetes objects with the same schemas.\n\n## Motivation\n\nWorkload portability is commonly cited as a goal for those deploying workloads. The Kubernetes API can provide a common API for deploying workloads across varying environments enabling some level of portability. For example, a deployment can be run in a cluster running on GKE, AKS, EKS, or clusters running elsewhere.\n\nBut, many applications rely on software as a service (SaaS). The reason for this is to push the operational details on to someone else who specializes in that particular service so the application developers and operators can focus on their application and business logic.\n\nThe problem is that one cannot deploy the same application in two different environments by two different providers, if the applications leverages services, with the same set of resources. This includes cases where the service being leveraged is common (e.g., MySQL as a Service). This problem limits application portability and sharing (e.g., in open source).\n\nThis KEP is looking to solve this problem by providing Kubernetes compatible objects, via CRDs and Secrets, that can be used in many environments by many providers to make working with common services easier. This can be used for services like database (e.g., MySQL, PostgreSQL), DNS, SMTP, and many others.\n\n### Goals\n\n* Provide a common way to request common services (e.g., MySQL)\n* Provide a common means to obtain credentials to use the service\n* Provide a common method to detect which services are available\n* Provide a system that can be implemented for the major public clouds, on-premise clusters, and local cluster (e.g., Docker for Mac)\n\n### Non-Goals\n\n* Provide an out of the box solution for every bespoke service provided by everyone\n* Replace Kubernetes Service Catalog\n\n## Proposal\n\n### User Stories\n\n#### Story 1\n\nAs a user of Kubernetes, I can query the services I can declaratively request using a Kubernetes native API. For example, using the command `kubectl get crds` where I can see services alongside other resources that can be created.\n\n#### Story 2\n\nAs a user of Kubernetes, I can declaratively request an instance of a service using a custom resource. When the service is provided the means to use that service (e.g., credentials in a secret) are provided in a common and consistent manner. The same resource and secret can be used in clusters running in different locations and the way the service is provided may be different.\n\n#### Story 3\n\nAs a cluster operator or application operator, I can discover controllers implementing the CRDs and secrets to support the application portability in my cluster.\n\n#### Story 4\n\nAs a cluster operator or application operator, I can set default values and provider custom settings for a service. \n\n### Implementation Details/Notes/Constraints\n\nTo solve the two user stories there are two types of Kubernetes resources that can be leveraged.\n\n1. Custom resource definitions (CRDs) can be used to describe a service. The CRDs can be implemented by controllers for different environments and the list of installed CRDs can be queried to see what is supported in a cluster\n2. Secrets with a specific type and schema can be used to handle credentials and other relevant information for services that have them (e.g., a database). Not all services will require a secret (e.g., DNS)\n\nThis subproject will list and document the resources and how controllers can implement them. This provides for interoperability including that for controllers and other tools, like validators, and a canonical listing held by a vendor neutral party.\n\nIn addition to the resources, this subproject will also provide a controller implementing the defined services to support testing, providing an example implementation, and to support other Kubernetes subprojects (e.g., Minikube). Controllers produced by this project are _not_ meant to be used in production.\n\n3rd party controllers implementing the CRDs and secrets can use a variety of methods to implement the service handling. This is where the Kubernetes Service Catalog can be an option. This subproject will not host or support 3rd party controllers but will list them to aide in users discovering them. This is in support of the 3rd user story.\n\nTo support custom settings for services by a service provider and to add the ability to add default settings (user story 4) we are considering a pattern of using CRDs for a controller with configuration on a cluster wide and namespace based level. An example of this in existence today is the cert manager issuer and cluster issuer resources. How to support this pattern will be worked out as part of the process in the next step of building a working system. This pattern is tentative until worked out in practice.\n\nThe next step is to work out the details on a common service and an initial process by which future services can be added. To work out the details we will start with MySQL and go through the process to make it work as managed by Kubernetes and each of the top three public clouds as defined by adoption. Public clouds and other cloud platforms following the top three are welcome to be involved in the process but are not required in the process.\n\nBefore this KEP can move to being implementable at least 2 services need to go through the process of being implemented to prove out the process elements of this system.\n\n### Risks and Mitigations\n\nTwo known risks include:\n\n1. The access controls and relationships between accounts and services. How will proper user and tenancy information be passed to clouds that require this form of information?\n2. Details required, when requesting a service, can vary between cloud providers that will implement this as a SaaS. How can that information be commonized or otherwise handled?\n\n## Graduation Criteria\n\nThe following are the graduation criteria:\n\n- 5 organizations have adopted using the portable service definitions\n- Service definitions for at least 3 services have been created and are in use\n- Documentation exists explaining how a controller implementer can use the CRDs and secrets to create a controller of their own\n- Service consumer documentation exists explaining how to use portable service definitions within their applications\n- A documented process for bringing a new service from suggestion to an implementable solution\n\n## Implementation History\n\n- the _Summary_, _Motivation_, _Proposal_ sections developed\n\n## Drawbacks [optional]\n\nWhy should this KEP _not_ be implemented.\n\n## Alternatives [optional]\n\nAn alternative is to modify the service catalog to leverage CRDs and return common secret credentials. Drawbacks to this are that it would be a complicated re-write to the service catalog, according to the service catalog team, and the solution would still require the open service broker (OSB) to be implemented in all environments (e.g., Minikube) even where simpler models (e.g., a controller) could be used instead. The solution proposed here could work with the service catalog in environments it makes sense and use other models in other environments. The focus here is more on the application operator experience working with services than all of the implementations required to power it.\n\n## Infrastructure Needed [optional]\n\nThe following infrastructure elements are needed:\n\n- A new subproject under SIG Apps for organizational purposes\n- A git repository, in the `kubernetes-sigs` organization, to host the CRD and Secrets schemas along with the Kubernetes provided controller\n- Testing infrastructure to continuously test the codebase\n"
+  },
+  {
+    "id": "8e82fda8d83063d208b59e3bafca38fb",
+    "title": "Implement maxUnavailable for StatefulSets",
+    "authors": ["@krmayankk"],
+    "owningSig": "sig-apps",
+    "participatingSigs": ["sig-apps"],
+    "reviewers": ["@janetkuo", "@kow3ns"],
+    "approvers": ["@janetkuo", "@kow3ns"],
+    "editor": "TBD",
+    "creationDate": "2018-12-29",
+    "lastUpdated": "2019-08-10",
+    "status": "implementable",
+    "seeAlso": ["n/a"],
+    "replaces": ["n/a"],
+    "supersededBy": ["n/a"],
+    "markdown": "\n# Implement maxUnavailable in StatefulSet\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Story 1](#story-1)\n  - [Implementation Details](#implementation-details)\n    - [API Changes](#api-changes)\n      - [Recommended Choice](#recommended-choice)\n    - [Implementation](#implementation)\n  - [Risks and Mitigations](#risks-and-mitigations)\n  - [Upgrades/Downgrades](#upgradesdowngrades)\n  - [Tests](#tests)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Drawbacks](#drawbacks)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThe purpose of this enhancement is to implement maxUnavailable for StatefulSet during RollingUpdate. \nWhen a StatefulSet’s `.spec.updateStrategy.type` is set to `RollingUpdate`, the StatefulSet controller \nwill delete and recreate each Pod in the StatefulSet. The updating of each Pod currently happens one at a time. With support for `maxUnavailable`, the updating will proceed `maxUnavailable` number of pods at a time. \n\n## Motivation\n\nConsider the following scenarios:-\n\n1. My containers publish metrics to a time series system. If I am using a Deployment, each rolling \nupdate creates a new pod name and hence the metrics published by this new pod starts a new time series \nwhich makes tracking metrics for the application difficult. While this could be mitigated, it requires \nsome tricks on the time series collection side. It would be so much better, If we could use a \nStatefulSet object so my object names doesnt change and hence all metrics goes to a single time series. This will be easier if StatefulSet is at feature parity with Deployments.\n2. My Container does some initial startup tasks like loading up cache or something that takes a lot of \ntime. If we used StatefulSet, we can only go one pod at a time which would result in a slow rolling \nupdate. If StatefulSet supported maxUnavailable with value greater than 1, it would allow for a faster \nrollout since a total of maxUnavailable number of pods could be loading up the cache at the same time.\n3. My Stateful clustered application, has followers and leaders, with followers being many more than 1. My application can tolerate many followers going down at the same time. I want to be able to do faster \nrollouts by bringing down 2 or more followers at the same time. This is only possible if StatefulSet \nsupports maxUnavailable in Rolling Updates.\n4. Sometimes I just want easier tracking of revisions of a rolling update. Deployment does it through \nReplicaSets and has its own nuances. Understanding that requires diving into the complicacy of hashing \nand how ReplicaSets are named. Over and above that, there are some issues with hash collisions which \nfurther complicate the situation(I know they were solved). StatefulSet introduced ControllerRevisions \nin 1.7 which are much easier to think and reason about. They are used by DaemonSet and StatefulSet for \ntracking revisions. It would be so much nicer if all the use cases of Deployments can be met in \nStatefulSet's and additionally we could track the revisions by ControllerRevisions. Another way of \nsaying this is, all my Deployment use cases are easily met by StatefulSet, and additionally I can enjoy\neasier revision tracking only if StatefulSet supported `maxUnavailable`.\n\nWith this feature in place, when using StatefulSet with maxUnavailable \u003e1, the user is making a \nconscious choice that more than one pod going down at the same time during rolling update, would not \ncause issues with their Stateful Application which have per pod state and identity. Other Stateful \nApplications which cannot tolerate more than one pod going down, will resort to the current behavior of one pod at a time Rolling Updates.\n\n### Goals\nStatefulSet RollingUpdate strategy will contain an additional parameter called `maxUnavailable` to \ncontrol how many Pods will be brought down at a time, during Rolling Update.\n\n### Non-Goals\nNA\n\n## Proposal\n\n### User Stories\n\n#### Story 1\nAs a User of Kubernetes, I should be able to update my StatefulSet, more than one Pod at a time, in a \nRollingUpdate manner, if my Stateful app can tolerate more than one pod being down, thus allowing my \nupdate to finish much faster. \n\n### Implementation Details\n\n#### API Changes\n\nFollowing changes will be made to the Rolling Update Strategy for StatefulSet.\n\n```go\n// RollingUpdateStatefulSetStrategy is used to communicate parameter for RollingUpdateStatefulSetStrategyType.\ntype RollingUpdateStatefulSetStrategy struct {\n\t// THIS IS AN EXISTING FIELD\n        // Partition indicates the ordinal at which the StatefulSet should be\n        // partitioned.\n        // Default value is 0.\n        // +optional\n        Partition *int32 `json:\"partition,omitempty\" protobuf:\"varint,1,opt,name=partition\"`\n\n\t// NOTE THIS IS THE NEW FIELD BEING PROPOSED\n\t// The maximum number of pods that can be unavailable during the update.\n        // Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%).\n        // Absolute number is calculated from percentage by rounding down.\n        // Defaults to 1.\n        // +optional\n        MaxUnavailable *intstr.IntOrString `json:\"maxUnavailable,omitempty\" protobuf:\"bytes,2,opt,name=maxUnavailable\"`\n\t\n\t...\n}\n```\n\n- By Default, if maxUnavailable is not specified, its value will be assumed to be 1 and StatefulSets \nwill follow their old behavior. This will also help while upgrading from a release which doesnt support maxUnavailable to a release which supports this field.\n- If maxUnavailable is specified, it cannot be greater than total number of replicas.\n- If maxUnavailable is specified and partition is also specified, MaxUnavailable cannot be greater than `replicas-partition`\n- If a partition is specified, maxUnavailable will only apply to all the pods which are staged by the \npartition. Which means all Pods with an ordinal that is greater than or equal to the partition will be \nupdated when the StatefulSet’s .spec.template is updated. Lets say total replicas is 5 and partition is set to 2 and maxUnavailable is set to 2. If the image is changed in this scenario, following\n  are the possible behavior choices we have:-\n\n  1. Pods with ordinal 4 and 3 will start Terminating at the same time(because of maxUnavailable). Once they are both running and ready, pods with ordinal 2 will start Terminating. Pods with ordinal 0 and 1 \nwill remain untouched due the partition. In this choice, the number of pods terminating is not always \nmaxUnavailable, but sometimes less than that. For e.g. if pod with ordinal 3 is running and ready but 4 is not, we still wait for 4 to be running and ready before moving on to 2. This implementation avoids \nout of order Terminations of pods.\n  2. Pods with ordinal 4 and 3 will start Terminating at the same time(because of maxUnavailable). When any of 4 or 3 are running and ready, pods with ordinal 2 will start Terminating. This could violate \nordering guarantees, since if 3 is running and ready, then both 4 and 2 are terminating at the same \ntime out of order. If 4 is running and ready, then both 3 and 2 are Terminating at the same time and no ordering guarantees are violated. This implementation, guarantees, that always there are maxUnavailable number of Pods Terminating except the last batch.\n  3. Pod with ordinal 4 and 3 will start Terminating at the same time(because of maxUnavailable). When 4 is running and ready, 2 will start Terminating. At this time both 2 and 3 are terminating. If 3 is \nrunning and ready before 4, 2 wont start Terminating to preserve ordering semantics. So at this time, \nonly 1 is unavailable although we requested 2.\n  4. Introduce a field in Rolling Update, which decides whether we want maxUnavailable with ordering or without ordering guarantees. Depending on what the user wants, this Choice can either choose behavior 1 or 3 if ordering guarantees are needed or choose behavior 2 if they dont care. To simplify this further\nPodManagementPolicy today supports `OrderedReady` or `Parallel`. The `Parallel` mode only supports scale up and tear down of StatefulSets and currently doesnt apply to Rolling Updates. So instead of coming up\nwith a new field, we could use the PodManagementPolicy to choose the behavior the User wants.\n\n        1. PMP=Parallel will now apply to RollingUpdate. This will choose behavior described in 2 above.\n           This means always maxUnavailable number of Pods are terminating at the same time except in \n           the last case and no ordering guarantees are provided.\n        2. PMP=OrderedReady with maxUnavailable can choose one of behavior 1 or 3. \n\nNOTE: The goal is faster updates of an application. In some cases , people would need both ordering \nand faster updates. In other cases they just need faster updates and they dont care about ordering as \nlong as they get identity.\n\nChoice 1 is simpler to reason about. It does not always have maxUnavailable number of Pods in \nTerminating state. It does not guarantee ordering within the batch of maxUnavailable Pods. The maximum \ndifference in the ordinals which are Terminating out of Order, cannot be more than maxUnavailable.\n\nChoice 2 always offers maxUnavailable number of Pods in Terminating state. This can sometime lead to \npods terminating out of order. This will always lead to the fastest rollouts. The maximum difference in the ordinals which are Terminating out of Order, can be more than maxUnavailable.\n\nChoice 3 always guarantees than no two pods are ever Terminating out of order. It sometimes does that, \nat the cost of not being able to Terminate maxUnavailable pods. The implementationg for this might be \ncomplicated.\n\nChoice 4 provides a choice to the users and hence takes the guessing out of the picture on what they \nwill expect. Implementing Choice 4 using PMP would be the easiest.\n\n##### Recommended Choice\n\nI recommend Choice 4, using PMP=Parallel for the first Alpha Phase. This would give the users fast \nrollouts without having them to second guess what the behavior should be. This choice also allows for \neasily extending the behavior with PMP=OrderedReady in future to choose either behavior 1 or 3.\n\n#### Implementation\n\nTBD: Will be updated after we have agreed on the semantics being discussed above.\n\nhttps://github.com/kubernetes/kubernetes/blob/v1.13.0/pkg/controller/statefulset/stateful_set_control.go#L504\n```go\n...\n\t // we compute the minimum ordinal of the target sequence for a destructive update based on the strategy.\n        updateMin := 0\n\tmaxUnavailable := 1\n        if set.Spec.UpdateStrategy.RollingUpdate != nil {\n                updateMin = int(*set.Spec.UpdateStrategy.RollingUpdate.Partition)\n\n\t\t// NEW CODE HERE\n\t\tmaxUnavailable, err = intstrutil.GetValueFromIntOrPercent(intstrutil.ValueOrDefault(set.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable, intstrutil.FromInt(1)), int(replicaCount), false)\n\t\tif err != nil {\n\t\t\treturn \u0026status, err\n\t\t}\n\t}\n\n\tvar unavailablePods []string\n\t// we terminate the Pod with the largest ordinal that does not match the update revision.\n\tfor target := len(replicas) - 1; target \u003e= updateMin; target-- {\n\n\t\t// delete the Pod if it is not already terminating and does not match the update revision.\n\t\tif getPodRevision(replicas[target]) != updateRevision.Name \u0026\u0026 !isTerminating(replicas[target]) {\n\t\t\tklog.V(2).Infof(\"StatefulSet %s/%s terminating Pod %s for update\",\n\t\t\t\tset.Namespace,\n\t\t\t\tset.Name,\n\t\t\t\treplicas[target].Name)\n\t\t\tif err := ssc.podControl.DeleteStatefulPod(set, replicas[target]); err != nil {\n\t\t\t\treturn \u0026status, err\n\t\t\t}\n\n\t\t\t// After deleting a Pod, dont Return From Here Yet.\n\t\t\t// We might have maxUnavailable greater than 1\n\t\t\tstatus.CurrentReplicas--\n\t\t}\n\n\t\t// wait for unhealthy Pods on update\n\t\tif !isHealthy(replicas[target]) {\n\t\t\t// If this Pod is unhealthy regardless of revision, count it in \n\t\t\t// unavailable pods\n\t\t\tunavailablePods = append(unavailablePods, replicas[target].Name)\n\t\t\tklog.V(4).Infof(\n\t\t\t\t\"StatefulSet %s/%s is waiting for Pod %s to update\",\n\t\t\t\tset.Namespace,\n\t\t\t\tset.Name,\n\t\t\t\treplicas[target].Name)\n\t\t}\n\t\t\n\t\t// NEW CODE HERE\n\t\t// If at anytime, total number of unavailable Pods exceeds maxUnavailable,\n\t\t// we stop deleting more Pods for Update\n\t\tif len(unavailablePods) \u003e= maxUnavailable {\n\t\t\tklog.V(4).Infof(\n\t\t\t\t\"StatefulSet %s/%s is waiting for unavailable Pods %v to update, max Allowed to Update Simultaneously %v\",\n\t\t\t\tset.Namespace,\n\t\t\t\tset.Name,\n\t\t\t\tunavailablePods,\n\t\t\t\tmaxUnavilable)\n\t\t\treturn \u0026status, nil\n\t\t}\n\n\t}\n...\n```\n\n### Risks and Mitigations\nWe are proposing a new field called `maxUnavailable` whose default value will be 1. In this mode, StatefulSet will behave exactly like its current behavior.\nIts possible we introduce a bug in the implementation. The mitigation currently is that is disabled by default in Alpha phase for people to try out and give\nfeedback. \nIn Beta phase when its enabled by default, people will only see issues or bugs when `maxUnavailable` is set to something greater than 1. Since people have\ntried this feature in Alpha, we would have time to fix issues.\n\n\n### Upgrades/Downgrades\n\n- Upgrades\n When upgrading from a release without this feature, to a release with maxUnavailable, we will set maxUnavailable to 1. This would give users the same default\n behavior they have to come to expect of in previous releases\n- Downgrades\n When downgrading from a release with this feature, to a release without maxUnavailable, there are two cases\n -- if maxUnavailable is greater than 1 -- in this case user  can see unexpected behavior(Find out what is the recommendation here(Warning, disable upgrade, drop field, etc? )\n -- if maxUnavailable is less than equal to 1 -- in this case user wont see any difference in behavior\n\n### Tests\n\n- maxUnavailable =1, Same behavior as today with PodManagementPolicy as `OrderedReady` or `Parallel`\n- Each of these Tests can be run in PodManagementPolicy = `OrderedReady` or `Parallel` and the Update\n  should happen at most maxUnavailable Pods at a time in ordered or parallel fashion respectively.\n- maxUnavailable greater than 1 without partition\n- maxUnavailable greater than replicas without partition\n- maxUnavailable greater than 1 with partition and staged pods less then maxUnavailable\n- maxUnavailable greater than 1 with partition and staged pods same as maxUnavailable\n- maxUnavailable greater than 1 with partition and staged pods greater than maxUnavailable\n- maxUnavailable greater than 1 with partition and maxUnavailable greater than replicas\n\n## Graduation Criteria\n\n- Alpha: Initial support for maxUnavailable in StatefulSets added. Disabled by default. \n- Beta:  Enabled by default with default value of 1. \n\n\n## Implementation History\n\n- KEP Started on 1/1/2019\n- Implementation PR and UT by 8/30\n\n## Drawbacks\n\nNA\n\n## Alternatives\n\n- Users who need StatefulSets stable identity and are ok with getting a slow rolling update will continue to use StatefulSets. Users who\nare not ok with a slow rolling update, will continue to use Deployments with workarounds for the scenarios mentioned in the Motivations\nsection.\n- Another alternative would be to use OnDelete and deploy your own Custom Controller on top of StatefulSet Pods. There you can implement \nyour own logic for deleting more than one pods in a specific order. This requires more work on the user but give them ultimate flexibility.\n\n"
+  },
+  {
+    "id": "9e472f84c991eec90414a44dd15fa457",
+    "title": "graduate-pod-disruption-budget-to-stable",
+    "authors": ["@bsalamat"],
+    "owningSig": "sig-apps",
+    "participatingSigs": ["sig-scheduling"],
+    "reviewers": ["@liggitt", "@kow3ns"],
+    "approvers": ["@kow3ns", "@liggitt"],
+    "editor": "TBD",
+    "creationDate": "2019-03-18",
+    "lastUpdated": "2019-03-18",
+    "status": "implementable",
+    "seeAlso": [""],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Graduate PodDisruptionBudget to stable\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n    - [Mutable PDBs](#mutable-pdbs)\n    - [Eviction of non-ready pods](#eviction-of-non-ready-pods)\n    - [Make the disruption controller more lenient for pods belonging to non-scale controllers](#make-the-disruption-controller-more-lenient-for-pods-belonging-to-non-scale-controllers)\n  - [Risks and Mitigations](#risks-and-mitigations)\n  - [Test Plan](#test-plan)\n    - [Existing Tests](#existing-tests)\n    - [Needed Tests](#needed-tests)\n  - [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\n[Pod Disruption Budget (PDB)](https://kubernetes.io/docs/tasks/run-application/configure-pdb/)\nis a Kubernetes API that limits the number of pods of a collection that are down simultaneously from voluntary disruptions.\n[Kubernetes eviction API](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/#the-eviction-api)\ntakes PDB into account when terminating pods. If PDB is\nviolated, the eviction API returns failure and does not delete the requested pod.\nThe feature was introduced in Kubernetes 1.4 and promoted to beta in 1.5.\nIt has been in beta for a long time. This document lays out the plan to promote\nit to stable.\n\n## Motivation\n\nPDB API has been stable and is an important feature that allows users to improve\nreliability of their critical workloads. This feature has been in beta for a\nlong time. We need to promote to stable version given that we plan to support it\nlong term.\n\n### Goals\n\n* Plan to promote PDB API to stable version.\n* List action items to mitigate potential risks of supporting mutable PDB.\n\n### Non-Goals\n\n* Making changes to the API fields or their meanings.\n\n## Proposal\n\n### Implementation Details/Notes/Constraints\n\n#### Mutable PDBs\n\nA mutable PDB object allows its `MinAvailable`, `MaxUnavailable`, and `Selector`\nfields to be modified by clients. Components that use PDB must watch such\nmodifications and use the updated values when making decisions.\n\nThis feature is implemented by [this PR](https://github.com/kubernetes/kubernetes/pull/69867).\n\n#### Eviction of non-ready pods\n\nThere are a couple of open issues where pods can't be evicted, even if they not Ready and Running \n(https://github.com/kubernetes/kubernetes/issues/72320 and https://github.com/kubernetes/kubernetes/issues/80389).\nThe root of this issue, is that the rules in the disruption controller for what is a healthy pod, and the rules\nin the Eviction API for when a pod can be evicted without looking at the PDB are not the same. This means a pod can\nbe considered unhealthy by the disruption controller so it does not count as healthy when computing `DisruptionsAllowed`,\nbut will still require `DisruptionsAllowed` to be larger than 0 for it to be evicted. Some strange situations can\narise from this. For example if we have a PDB with `MinAvailable  = 1` and 10 pods that are all in the CrashLoop state \n(`Running`, but not `Ready`), we will not be allowed to evict any of the pods.\n\n#### Make the disruption controller more lenient for pods belonging to non-scale controllers\n\nThe disruption controller is currently taking the very safe route whenever it encounters any\nissues with the targeted pods or their controllers. For all configurations of the PDB, except when\n`minAvailable` is a number, the PDB requires that it can find the controller and that\nthe controller implements scale (either by being one of the built-in workloads or a CR where the CRD implements\nthe scale subresource). If those conditions are not met for all pods targeted by the PDB, the disruption\ncontroller will set `DisruptionsAllowed` to 0, which means none of the pods can be evicted. There is an issue\nconcerning this behavior: https://github.com/kubernetes/kubernetes/issues/77383.\n\nThe current behavior of the disruption controller for the different types of input and the different\ntypes of pods that might be encountered are documented in: \nhttps://docs.google.com/spreadsheets/d/12HUundBS-slA6axfQYZPRCeIu_Au_wsGD0Vu_oKAnM8/edit?usp=sharing\n\n### Risks and Mitigations\n\nWe plan to support mutation of PDB objects that didn't exist in previous versions.\nThe following needs to be checked and verified before graduation of the API.\n\n- [ ] Ensure that components do not have any logic that relies on immutability\nof PDBs. For example, if component builds a cache of pods that match various\nPDBs, they must add logic to invalid the cache on updates.\n   - Action Item: sweep for in-tree (including kubernetes/* repos) uses to make\n   sure components are informer driven and aren’t assuming immutability.\n- [ ] Check what watches PDBs and make sure there is no performance concerns.\n- [ ] Updating a PDB could cause the state of a cluster to seem incorrect. For\nexample, a PDB states that at least 10 replicas of a collection must be alive.\nKubernetes control plane evicts some of the existing pods, but keeps at least 10\naround. The PDB is updated and states that at least 20 replicas must be kept\nalive. It may appear to an observer that the evictions happened before the PDB \nupdate were incorrect, if they don’t notice the PDB update.\n  - Action Item: Update documents and explain mutability of PDB its side effects.\n\n### Test Plan\n\n#### Existing Tests\nPodDisruptionBudget currently has tests in various components that use the feature:\n\n* Disruption controller\n  - [This test](https://github.com/kubernetes/kubernetes/blob/687d759e362b05dcdf11e336e2799704918e048d/pkg/controller/disruption/disruption_test.go#L140)\n  tests PDB MinAvailable, MaxUnavailable, and selector functionality in Disruption controller.\n* Kubectl\n  - [This test](https://github.com/kubernetes/kubernetes/blob/master/pkg/kubectl/generate/versioned/pdb_test.go)\n  tests generation of a PDB objects out of given parameters\n  - [This test](https://github.com/kubernetes/kubernetes/blob/master/pkg/kubectl/cmd/create/create_pdb_test.go)\n  tests creation of PDB objects from cmd parameters\n* Scheduler\n  - [This test](https://github.com/kubernetes/kubernetes/blob/ac56bd502ab96696682c66ebdff94b6e52471aa3/test/integration/scheduler/preemption_test.go#L731)\n  tests effects of PDB on preemption (PDB is honored in a best effort way)\n* Eviction integration tests\n  - [These tests](https://github.com/kubernetes/kubernetes/blob/master/test/integration/evictions/evictions_test.go) test eviction logic and its interactions with PDB.\n* Autoscaler\n  - [These tests](https://github.com/kubernetes/kubernetes/blob/master/test/e2e/autoscaling/cluster_size_autoscaling.go) ensure that Autoscaler respects PDB when draining nodes.\n\n#### Needed Tests\n\nAs a non-optional feature, there should be a conformance test for\nPodDisruptionBudget.\n\n\n### Graduation Criteria\n\n- [ ] Implement Mutable PDBs\n- [ ] Needs a conformance test\n- [ ] Update documents to reflect the changes\n\n## Implementation History\n\n- PodDisruptionBudget was introduced in Kubernetes 1.4 as an alpha version.\n- PodDisruptionBudget was graduated to beta in Kubernetes 1.5.\n\n"
+  },
+  {
+    "id": "cfd983cf9debad6c2eef91cc3d534c74",
+    "title": "pdb-support-for-custom-resources-with-scale-subresource",
+    "authors": ["@mortent"],
+    "owningSig": "sig-apps",
+    "participatingSigs": ["sig-scheduling", "sig-autoscaling"],
+    "reviewers": ["@kow3ns", "@janetkuo"],
+    "approvers": ["@kow3ns", "@janetkuo"],
+    "editor": "TBD",
+    "creationDate": "2019-04-12",
+    "lastUpdated": "2019-04-12",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# PDB support for custom resources with scale subresource\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n- [Proposal](#proposal)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\n[Pod Disruption Budget (PDB)](https://kubernetes.io/docs/tasks/run-application/configure-pdb/)\nis a Kubernetes API that limits the number of pods of a collection that are down simultaneously from voluntary disruptions. PDBs allows a user to specify the allowed disruption through either min available or max unavailable number of pods. In order to support PDBs where max number of unavailable pods is set, the PDB controller needs to be able to look up the desired number of replicas. It does this by looking at the controller(s) (as specified by the owner ref) for the pods covered by the PDB. Currently only four of the basic workload controllers are supported for PDBs, namely Deployment, StatefulSet, ReplicaSet and ReplicationController. \n\nThe scale subresource allows any resource to specify its desired number of replicas and a generic way to look up this information. This document lays out a plan to use the scale subresource to allow setting PDBs for any resource implementing the scale subresource.\n\n## Motivation\n\nPDBs are an important tool to control the number of voluntary disruptions for workloads on Kubernetes. As more users start deploying custom controllers/operators based on CRDs (EtcdCluster, MySQLReplicaSet...), it is inconvenient that they cannot take advantage of PDBs. This doesn't work today because the PDB controller needs to know the desired number of replicas specified in a controller and the PDB controller only knows how to find this from the four Kubernetes workload controllers mentioned above. The scale subresource is already in use by the autoscaler and it provides a generic way to look up the desired number of replicas from any custom resource with a scale subresource. We can leverage this to support PDBs for custom controllers.\n\n### Goals\n\n- Implement support for scale subresources in PDBs\n- Avoid major performance impact from the change\n\n## Proposal\n\n### Implementation Details/Notes/Constraints\n\nIn the current implementation, the PDB controller identifies the workload controller by going through all the pods covered by the PDB, and for each pod, check in sequence for each of the four workload controllers. Each check looks at the controller reference, and if the kind is correct, looks up the controller from the shared informer. The PDB controller then looks up the desired number of replicas from the identified controller. If the set of pods covered by the PDB identfies more than one controller, that is considered an error. \n\nThe plan is to keep this approach, but check for the scale subresource if neither of the kubernetes workload controllers match. Each of the kubernetes workload controllers actually implement the scale subresource, so it is possible to only use that one. But since shared informers can not be used with the scale subresource, we want to rely on the informers for checking the kubernetes workload controllers, and only try the scale subresource if none of the them matches. This way the controller will only need to hit the apiserver in the less likely scenario where the pods have a controller other than the standard kubernetes workload controllers.\n\nAs mentioned above, the PDB controller will check the desired number of replicas from the controller for each pod covered by the PDB. In almost all scenarios the controller will be the same for all pods and therefore the desired number of replicas will be the same. This is not a concern for the kubernetes workload controllers since the controller uses shared informers. When using the scale subresource, this would lead to unnecessary calls to the apiserver. We will implement a per reconcile loop cache, so the PDB controller only needs to look up each controller once per reconcile loop.\n\n### Risks and Mitigations\n\nThe major risk with this change is the additional load on the apiserver since we can't use shared informers for scale subresources. As described above, we plan to mitigate this by only checking the scale subresource when the kind of the controller ref doesn't match any of the kubernetes workload controllers, and use a cache to make sure we only need to look up each controller once per reconcile loop.\n\n## Design Details\n\n### Test Plan\n\n* Unit tests covering the usage of PDBs with custom resources that is implementing the scale subresource.\n* Integration tests to make sure using the scale subresource endpoint from the PDB controller works as expected.\n\n### Graduation Criteria\n\nThis will be added as a beta enhancement to PDBs. It doesn't change the existing API or behvior but only adds an additional code path to handle non built-in types.\n\n[KEP](https://github.com/kubernetes/enhancements/pull/904) for graduating PDBs to GA is already underway. It involves a change to make PDBs mutable. [PR](https://github.com/kubernetes/kubernetes/pull/69867) for this is almost ready to merge. The goal is to get both that change and this one into the next version of Kubernetes (1.15), and unless any serious issues come up, promote PDBs to GA the following release (1.16).\n\n## Implementation History\n\n- Initial PR: https://github.com/kubernetes/kubernetes/pull/76294\n\n"
+  },
+  {
+    "id": "26dca20167ab49f88239e60184c8e012",
+    "title": "Sidecar Containers",
+    "authors": ["@joseph-irving"],
+    "owningSig": "sig-apps",
+    "participatingSigs": ["sig-apps", "sig-node"],
+    "reviewers": ["@fejta"],
+    "approvers": ["@enisoc", "@kow3ns"],
+    "editor": "TBD",
+    "creationDate": "2018-05-14",
+    "lastUpdated": "2019-10-30",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Sidecar Containers\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Jobs](#jobs)\n  - [Startup](#startup)\n  - [Shutdown](#shutdown)\n- [Goals](#goals)\n- [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n    - [API Changes:](#api-changes)\n    - [Kubelet Changes:](#kubelet-changes)\n      - [Shutdown triggering](#shutdown-triggering)\n      - [Sidecars terminated last](#sidecars-terminated-last)\n      - [Sidecars started first](#sidecars-started-first)\n      - [PreStop hooks sent to Sidecars first](#prestop-hooks-sent-to-sidecars-first)\n    - [PoC and Demo](#poc-and-demo)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n**ACTION REQUIRED:** In order to merge code into a release, there must be an issue in [kubernetes/enhancements] referencing this KEP and targeting a release milestone **before [Enhancement Freeze](https://github.com/kubernetes/sig-release/tree/master/releases)\nof the targeted release**.\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e., [kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be updated for the enhancement to be released.\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [X] KEP approvers have set the KEP status to `implementable`\n- [X] Design details are appropriately documented\n- [X] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [X] Graduation criteria is in place\n- [X] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nTo solve the problem of container lifecycle dependency we can create a new class of container: a \"sidecar container\" that behaves primarily like a normal container but is handled differently during termination and startup.\n\n## Motivation\n\nSideCar containers have always been used in some ways but just not formally identified as such, they are becoming more common in a lot of applications and as more people have used them, more issues have cropped up.\n\nHere are some examples of the main problems:\n\n### Jobs\n If you have a Job with two containers one of which is actually doing the main processing of the job and the other is just facilitating it, you encounter a problem when the main process finishes; your sidecar container will carry on running so the job will never finish.\n\nThe only way around this problem is to manage the sidecar container's lifecycle manually and arrange for it to exit when the main container exits. This is typically achieved by building an ad-hoc signalling mechanism to communicate completion status between containers. Common implementations use a shared scratch volume mounted into all pods, where lifecycle status can be communicated by creating and watching for the presence of files. This pattern has several disadvantages:\n\n* Repetitive lifecycle logic must be rewritten in each instance a sidecar is deployed.\n* Third-party containers typically require a wrapper to add this behaviour, normally provided via an entrypoint wrapper script implemented in the k8s container spec. This adds undesirable overhead and introduces repetition between the k8s and upstream container image specs.\n* The wrapping typically requires the presence of a shell in the container image, so this pattern does not work for minimal containers which ship without a toolchain.\n\n### Startup\nAn application that has a proxy container acting as a sidecar may fail when it starts up as it's unable to communicate until its proxy has started up successfully. Readiness probes don't help if the application is trying to talk outbound.\n\n### Shutdown\nApplications that rely on sidecars may experience a high amount of errors when shutting down as the sidecar may terminate before the application has finished what it's doing.\n\n\n## Goals\n\nSolve issues so that they don't require application modification:\n* [25908](https://github.com/kubernetes/kubernetes/issues/25908) - Job completion\n* [65502](https://github.com/kubernetes/kubernetes/issues/65502) - Container startup dependencies\n\n## Non-Goals\n\nAllowing multiple containers to run at once during the init phase - this could be solved using the same principal but can be implemented separately. //TODO write up how we could solve the init problem with this proposal\n\n## Proposal\n\nCreate a way to define containers as sidecars, this will be an additional field to the `container.lifecycle` spec: `Type` which can be either `Standard` (default) or `Sidecar`.\n\ne.g:\n```yaml\napiVersion: v1\nkind: Pod\nmetadata:\n  name: myapp-pod\n  labels:\n    app: myapp\nspec:\n  containers:\n  - name: myapp\n    image: myapp\n    command: ['do something']\n  - name: sidecar\n    image: sidecar-image\n    lifecycle:\n      type: Sidecar\n    command: [\"do something to help my app\"]\n\n```\nSidecars will be started before normal containers but after init, so that they are ready before your main processes start.\n\nThis will change the Pod startup to look like this:\n* Init containers start\n* Init containers finish\n* Sidecars start\n* Sidecars become ready\n* Containers start\n\nDuring pod termination sidecars will be terminated last:\n* Containers sent SIGTERM\n* Once all Containers have exited: Sidecars sent SIGTERM\n\nContainers and Sidecar will share the TerminationGracePeriod. If Containers don't exit before the end of the TerminationGracePeriod then they will be sent a SIGKIll as normal, Sidecars will then be sent a SIGTERM with a short grace period of 2 Seconds to give them a chance to cleanly exit.\n\nPreStop Hooks will be sent to sidecars before containers are terminated.\nThis will be useful in scenarios such as when your sidecar is a proxy so that it knows to no longer accept inbound requests but can continue to allow outbound ones until the the primary containers have shut down.\n\nTo solve the problem of Jobs that don't complete: When RestartPolicy!=Always if all normal containers have reached a terminal state (Succeeded for restartPolicy=OnFailure, or Succeeded/Failed for restartPolicy=Never), then all sidecar containers will be sent a SIGTERM.\n\nPodPhase will be modified to not include Sidecars in its calculations, this is so that if a sidecar exits in failure it does not mark the pod as `Failed`. It also avoids the scenario in which a Pod has RestartPolicy `OnFailure`, if the containers all successfully complete, when the sidecar gets sent the shut down signal if it exits with a non-zero code the Pod phase would be calculated as `Running` despite all containers having exited permanently.\n\nSidecars are just normal containers in almost all respects, they have all the same attributes, they are included in pod state, obey pod restart policy etc. The only differences are lifecycle related.\n\n### Implementation Details/Notes/Constraints\n\nThe proposal can broken down into four key pieces of implementation that all relatively separate from one another:\n\n* Shutdown triggering for sidecars when RestartPolicy!=Always\n* Pre-stop hooks sent to sidecars before non sidecar containers\n* Sidecars are terminated after normal containers\n* Sidecars start before normal containers\n\n#### API Changes:\nAs this is a change to the Container spec we will be using feature gating, you will be required to explicitly enable this feature on the api server as recommended [here](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api_changes.md#adding-unstable-features-to-stable-versions).\n\nNew field `Type` will be added to the lifecycle struct:\n\n```go\ntype Lifecycle struct {\n  // Type\n  // One of Standard, Sidecar.\n  // Defaults to Standard\n  // +optional\n  Type LifecycleType `json:\"type,omitempty\" protobuf:\"bytes,3,opt,name=type,casttype=LifecycleType\"`\n}\n```\nNew type `LifecycleType` will be added with two constants:\n```go\n// LifecycleType describes the lifecycle behaviour of the container\ntype LifecycleType string\n\nconst (\n  // LifecycleTypeStandard is the default container lifecycle behaviour\n  LifecycleTypeStandard LifecycleType = \"Standard\"\n  // LifecycleTypeSidecar means that the container will start up before standard containers and be terminated after\n  LifecycleTypeSidecar LifecycleType = \"Sidecar\"\n)\n```\nNote that currently the `lifecycle` struct is only used for `preStop` and `postStop` so we will need to change its description to reflect the expansion of its uses.\n\n#### Kubelet Changes:\nBroad outline of what places could be modified to implement desired behaviour:\n\n##### Shutdown triggering\nPackage `kuberuntime`\n\nModify `kuberuntime_manager.go`, function `computePodActions`. Have a check in this function that will see if all the non-sidecars had permanently exited, if true: return all the running sidecars in `ContainersToKill`. These containers will then be killed via the `killContainer` function which sends preStop hooks, sig-terms and obeys grace period, thus giving the sidecars a chance to gracefully terminate.\n\n##### Sidecars terminated last\nPackage `kuberuntime`\n\nModify `kuberuntime_container.go`, function `killContainersWithSyncResult`. Break up the looping over containers so that it goes through killing the non-sidecars before terminating the sidecars.\nNote that the containers in this function are `kubecontainer.Container` instead of `v1.Container` so we would need to cross reference with the `v1.Pod` to check if they are sidecars or not. This Pod can be `nil` but only if it's not running, in which case we're not worried about ordering.\n\n##### Sidecars started first\nPackage `kuberuntime`\n\nModify `kuberuntime_manager.go`, function `computePodActions`. If pods has sidecars it will return these first in `ContainersToStart`, until they are all ready it will not return the non-sidecars. Readiness changes do not normally trigger a pod sync, so to avoid waiting for the Kubelet's `SyncFrequency` (default 1 minute) we can modify `HandlePodReconcile` in the `kubelet.go` to trigger a sync when the sidecars first become ready (ie only during startup).\n\n##### PreStop hooks sent to Sidecars first\nPackage `kuberuntime`\n\nModify `kuberuntime_container.go`, function `killContainersWithSyncResult`. Loop over sidecars and execute `executePreStopHook` on them before moving on to terminating the containers. This approach would assume that PreStop Hooks are idempotent as the sidecars would get sent the PreStop hook again when they are terminated.\n\n#### PoC and Demo\nThere is a [PR here](https://github.com/kubernetes/kubernetes/pull/75099) with a working Proof of concept for this KEP, it's just a draft but should help illustrate what these changes would look like.\n\nPlease view this [video](https://youtu.be/4hC8t6_8bTs) if you want to see what the PoC looks like in action.\n\n### Risks and Mitigations\n\nYou could set all containers to have `lifecycle.type: Sidecar`, this would cause strange behaviour in regards to shutting down the sidecars when all the non-sidecars have exited. To solve this the api could do a validation check that at least one container is not a sidecar.\n\nInit containers would be able to have `lifecycle.type: Sidecar` applied to them as it's an additional field to the container spec, this doesn't currently make sense as init containers are ran sequentially. We could get around this by having the api throw a validation error if you try to use this field on an init container or just ignore the field.\n\nOlder Kubelets that don't implement the sidecar logic could have a pod scheduled on them that has the sidecar field. As this field is just an addition to the Container Spec the Kubelet would still be able to schedule the pod, treating the sidecars as if they were just a normal container. This could potentially cause confusion to a user as their pod would not behave in the way they expect, but would avoid pods being unable to schedule.\n\nShutdown ordering of Containers in a Pod can not be guaranteed when a node is being shutdown, this is due to the fact that the Kubelet is not responsible for stopping containers when the node shuts down, it is instead handed off to systemd (when on Linux) which would not be aware of the ordering requirements. Daemonset and static Pods would be the most effected as they are typically not drained from a node before it is shutdown. This could be seen as a larger issue with node shutdown (also effects things like termination grace period) and does not necessarily need to be addressed in this KEP , however it should be clear in the documentation what guarantees we can provide in regards to the ordering.\n\n## Design Details\n\n### Test Plan\n* Units test in kubelet package `kuberuntime` primarily in the same style as `TestComputePodActions` to test a variety of scenarios.\n* New E2E Tests to validate that pods with sidecars behave as expected e.g:\n * Pod with sidecars starts sidecars containers before non-sidecars\n * Pod with sidecars terminates non-sidecar containers before sidecars\n * Pod with init containers and sidecars starts sidecars after init phase, before non-sidecars\n * Termination grace period is still respected when terminating a Pod with sidecars\n * Pod with sidecars terminates sidecars once non-sidecars have completed when `restartPolicy` != `Always`\n * Pod phase should be `Failed` if any sidecar exits in failure when `restartPolicy` != `Always`\n * Pod phase should be `Succeeded` if all containers, including sidecars, exit with success when `restartPolicy` != `Always`\n\n\n### Graduation Criteria\n#### Alpha -\u003e Beta Graduation\n* Addressed feedback from Alpha testers\n* Thorough E2E and Unit testing in place\n* The beta API either supports the important use cases discovered during alpha testing, or has room for further enhancements that would support them\n\n#### Beta -\u003e GA Graduation\n* Sufficient number of end users are using the feature\n* We're confident that no further API changes will be needed to achieve the goals of the KEP\n* All known blocking bugs have been fixed\n\n### Upgrade / Downgrade Strategy\nWhen upgrading no changes should be needed to maintain existing behaviour as all of this behaviour is optional and disabled by default.\nTo activate the feature they will need to enable the feature gate and mark their containers as sidecars in the container spec.\n\nWhen downgrading `kubectl`, users will need to remove the sidecar field from any of their Kubernetes manifest files as `kubectl` will refuse to apply manifests with an unknown field (unless you use `--validate=false`).\n\n### Version Skew Strategy\nOlder Kubelets should still be able to schedule Pods that have sidecar containers however they will behave just like a normal container.\n\n## Implementation History\n\n- 14th May 2018: Proposal Submitted\n- 26th June 2019: KEP Marked as implementable\n\n## Alternatives\n\nOne alternative would be to have a new field in the Pod Spec of `sidecarContainers:` where you could define a list of sidecar containers, however this would require more work in terms of updating tooling/kubelet to support this.\n\nAnother alternative would be to change the Job Spec to have a `primaryContainer` field to tell it which containers are important. However I feel this is perhaps too specific to job when this Sidecar concept could be useful in other scenarios.\n\nA boolean flag of `sidecar: true` could be used to indicate which pods are sidecars, however this prevents additional ContainerLifecycles from being added in the future.\n"
+  },
+  {
+    "id": "47eab0e5a3ce48b6d0406d55a9bf858b",
+    "title": "go modules",
+    "authors": ["@liggitt"],
+    "owningSig": "sig-architecture",
+    "participatingSigs": ["sig-api-machinery", "sig-release", "sig-testing"],
+    "reviewers": ["@sttts", "@lavalamp"],
+    "approvers": ["@smarterclayton", "@thockin"],
+    "editor": "",
+    "creationDate": "2019-03-19",
+    "lastUpdated": "2019-11-01",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# go modules\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n- [Proposal](#proposal)\n  - [Manage vendor folders using go modules](#manage-vendor-folders-using-go-modules)\n  - [Publish staging component modules to individual repositories](#publish-staging-component-modules-to-individual-repositories)\n  - [Select a versioning strategy for published modules](#select-a-versioning-strategy-for-published-modules)\n  - [Remove Godeps](#remove-godeps)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n  - [Alternatives to vendoring using go modules](#alternatives-to-vendoring-using-go-modules)\n  - [Alternatives to publishing staging component modules](#alternatives-to-publishing-staging-component-modules)\n  - [Alternative versioning strategies](#alternative-versioning-strategies)\n- [Reference](#reference)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n**ACTION REQUIRED:** In order to merge code into a release, there must be an issue in [kubernetes/enhancements] referencing this KEP and targeting a release milestone **before [Enhancement Freeze](https://github.com/kubernetes/sig-release/tree/master/releases)\nof the targeted release**.\n\nThese checklist items _must_ be updated for the enhancement to be released.\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP: https://github.com/kubernetes/enhancements/issues/917\n- [x] KEP approvers have set the KEP status to `implementable`\n- [x] Design details are appropriately documented\n- [x] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [x] Graduation criteria is in place\n- [x] \"Implementation History\" section is up-to-date for milestone\n- [x] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n- [x] User-facing documentation has been created\n\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n\n## Summary\n\nManage the vendor folder in `kubernetes/kubernetes` using go modules,\nand define `go.mod` module files for published components like `k8s.io/client-go` and `k8s.io/api`.\n\n## Motivation\n\nSince its inception, Kubernetes has used Godep to manage vendored \ndependencies, to ensure reproducible builds and auditable source.\n\nAs the go ecosystem matured, vendoring became a first-class concept,\nGodep became unmaintained, Kubernetes started using a custom version of Godep,\nother vendoring tools (like `glide` and `dep`) became available,\nand dependency management was ultimately added directly to go in the form of go modules.\n\nThe [plan of record](https://blog.golang.org/modules2019) is for go1.13 to enable\ngo modules by default and deprecate GOPATH mode. To be ready for that, Kubernetes\ncomponents must make any required adjustments in the Kubernetes 1.15 release.\n\nIn addition to simply keeping up with the go ecosystem, go modules provide many benefits:\n* rebuilding `vendor` with go modules provided a 10x speed increase over Godep in preliminary tests\n* go modules can reproduce a consistent `vendor` directory on any OS\n* if semantic import versioning is adopted, consumers of Kubernetes modules can use two distinct versions simultaneously (if required by diamond dependencies)\n\n### Goals\n\n* Remove the kubernetes/kubernetes dependency on godep (including the need for a customized version of godep)\n* Increase speed and reliability of building and verifying the `vendor` directory in kubernetes/kubernetes\n* Provide accurate go.mod module descriptors for kubernetes/kubernetes and published staging components\n* Enable use of published kubernetes components by `go module` aware consumers\n* Allow continued use of published kubernetes components by `go module` *unaware* consumers until go modules are enabled by default\n* Allow consumers of go modules to understand the versions of kubernetes libraries they are using\n\n## Proposal\n\n### Manage vendor folders using go modules\n1. Generate `go.mod` files for `k8s.io/kubernetes` and each staging component (e.g. `k8s.io/client-go` and `k8s.io/api`) as a distinct go module\n  \n    * Add `require` and `replace` directives in all the `go.mod` files to register a preference for the same dependency versions currently listed in `Godeps.json`\n\n        ```\n        require (\n          bitbucket.org/bertimus9/systemstat v0.0.0-20180207000608-0eeff89b0690\n          ...\n        )\n        replace bitbucket.org/bertimus9/systemstat =\u003e bitbucket.org/bertimus9/systemstat v0.0.0-20180207000608-0eeff89b0690\n        ...\n        ```\n\n    * Add `require` and `replace` directives in the `k8s.io/kubernetes` `go.mod` file to point to the staging component directories:\n\n        ```\n        require (\n          k8s.io/api v0.0.0\n          ...\n        )\n        replace k8s.io/api =\u003e ./staging/src/k8s.io/api\n        ...\n        ```\n\n    * Add `require` and `replace` directives in the staging component `go.mod` files to point to the peer component directories:\n\n        ```\n        require (\n          k8s.io/api v0.0.0\n          ...\n        )\n        replace k8s.io/api =\u003e ../api\n        ...\n        ```\n\n2. Change vendor creation and verification scripts in `kubernetes/kubernetes` to use go module commands to:\n\n    * Sync dependency versions between `k8s.io/kubernetes` and in staging component `go.mod` files\n    * Build the vendor directory (`go mod vendor`)\n    * Generate the vendored `LICENSES` file\n\nSee the [alternatives](#alternatives-to-vendoring-using-go-modules) section for other vendoring tools considered.\n\n### Publish staging component modules to individual repositories\n\nUpdate the staging component publishing bot:\n1. Rewrite the staging component `require` directives in `go.mod` files to require specific published versions (the same thing done today for `Godeps.json` files)\n2. Remove the staging component `replace` directives in `go.mod` files (relative path references don't make sense for independent repositories)\n3. Stop including `vendor` content in published modules. Vendor folders are ignored in non-top-level modules, and published `go.mod` files inform `go get` of desired versions of transitive dependencies.\n4. Generate synthetic `Godeps.json` files containing the SHA or git tag of module dependencies,\nfor consumption by downstream consumers using dependency management tools like `glide`.\nContinue publishing these at least until our minimum supported version of go defaults to enabling\nmodule support (currently targeted for go 1.13, which is approximately Kubernetes 1.16-1.17 timeframe).\n\nSee the [alternatives](#alternatives-to-publishing-staging-component-modules) section for other publishing methods considered.\n\n### Select a versioning strategy for published modules\n\nAs part of transitioning to go modules, we can select a versioning strategy.\n\nState prior to adoption of go modules:\n* `client-go` tagged a major version on every kubernetes release\n* all components tagged a (non-semver) `kubernetes-1.x.y` version on each release\n* no import rewriting occurred\n* consumers could only make use of a single version of each component in a build\n\nState after adoption of go modules:\n* all components tagged a (non-semver) `kubernetes-1.x.y` version on each release\n* no import rewriting occurred\n* consumers could only make use of a single version of each component in a build\n\nThis proposes publishing components with the following tags:\n* Non-semver tags of `kubernetes-1.x.y` (corresponding to kubernetes `v1.x.y`)\n* Semver tags of `v0.x.y` (corresponding to kubernetes `v1.x.y`)\n\n`v0.x.y` accurately convey the current guarantees around the go APIs release-to-release.\nThe semver tags are preserved in the go.mod files of consuming components, \nallowing them to see what versions of kubernetes libraries they are using.\nWithout semver tags, downstream components see \"pseudo-versions\" like\n`v0.0.0-20181208010431-42b417875d0f` in their go.mod files, making it \nextremely difficult to see if there are version mismatches between the \nkubernetes libraries they are using.\n\nThis results in the following usage patterns:\n\nConsumers:\n* GOPATH consumers\n  * import `k8s.io/apimachinery/...` (as they do today)\n  * `go get` behavior (e.g. `go get client-go`):\n    * uses latest commits of transitive `k8s.io/...` dependencies (likely to break, as today)\n    * uses latest commits of transitive non-module-based dependencies (likely to break, as today)\n* module-based consumers using a specific version\n  * reference published module versions as `v0.x.y` or `kubernetes-1.x.y`\n  * import `k8s.io/apimachinery/...` (as they do today)\n  * `go get` behavior (e.g. `GO111MODULE=on go get k8s.io/client-go@v0.17.0` or `GO111MODULE=on go get k8s.io/client-go@kubernetes-1.17.0`):\n    * uses `go.mod`-referenced versions of transitive `k8s.io/...` dependencies (unless overridden by top-level module, or conflicting peers referencing later versions)\n    * uses `go.mod`-referenced versions of transitive non-module-based dependencies (unless overridden by top-level module, or conflicting peers referencing later versions)\n* consumers are limited to a single copy of kubernetes libraries among all dependencies (as they are today)\n\nKubernetes tooling:\n* minimal changes required\n\nCompatibility implications:\n* breaking go changes in each release impact consumers that have not pinned to particular tags/shas (as they do today)\n* conflicting version requirements (direct or transitive) can result in impossible-to-build or impossible-to-update dependencies (as they do today)\n\nThis would not limit future changes to our versioning strategy:\n* modules published and tagged this way could transition to semantic import versioning in the future, if desired\n* modules published and tagged this way could transition to v1.x.y semver tagging in the future, if desired\n  (this would require enforcement of go API compatibility in perpetuity, and prevent removal of *any* go API element,\n  so we are unlikely to pursue this approach, but adding v0.x.y tags now does not remove the option)\n\nSee the [alternatives](#alternative-versioning-strategies) section for other possible versioning strategies considered for the initial move to modules.\n\n### Remove Godeps\n\n* Move aggregated `Godeps/LICENSES` file to `vendor/LICENSES` (and ensure it is packaged correctly in build artifacts)\n* Remove `Godeps.json` files from `kubernetes/kubernetes` and staging component directories.\nWith the change to go modules, the only use for these is as a hint to non-module-based downstream consumers of the published staging components.\n* Remove the custom `Godeps` fork from `kubernetes/kubernetes`\n* Remove all other `Godeps` references in scripts, comments, and configurations files\n\n## Design Details\n\n### Test Plan\n\n* CI scripts to verify vendor contents are recreatable and match referenced versions\n* CI scripts to verify vendor licenses are up to date\n* CI scripts to verify staging component dependencies are correct\n* CI scripts to verify staging component publishing is successful\n* CI scripts to verify examples using `k8s.io/client-go` can be consumed and build automatically and successfully by GOPATH and module-based consumers\n\n### Graduation Criteria\n\n* `k8s.io/kubernetes` vendor management uses go modules\n  * CI verifies vendor management scripts succeed with `GOPATH` unset, in a directory structure not shaped like `$GOPATH`\n* there are documented processes for:\n  * adding/pinning a new dependency\n  * updating the pinned version of an existing dependency\n  * removing an unnecessary dependency\n* published staging components can be successfully consumed by:\n  * go module consumers using `go get`\n  * GOPATH-based consumers using `go get`\n\n### Upgrade / Downgrade Strategy\n\nNot applicable\n\n### Version Skew Strategy\n\nNot applicable\n\n## Implementation History\n\n- 2019-03-19: Created\n- 2019-03-26: Completed proposal\n- 2019-03-26: Marked implementable\n- 2019-04-03: Implemented go module support\n- 2019-11-01: Added proposal for tagging published modules with v0.x.y\n\n## Alternatives\n\n### Alternatives to vendoring using go modules\n\n* Continue using `godep` for vendor management. This is not viable for several reasons:\n  * The tool is unmaintained (the project readme states \"Please use dep or another tool instead.\"), and we have had to make our own fork to work around some edge cases.\n  * There are significant performance problems (the pull-kubernetes-godeps CI job takes ~30 minutes to verify vendoring)\n  * There are significant functional problems (the tool cannot be run in some environments, gets confused by diamond dependencies, etc)\n\n* Use an alternate dependency mechanism (e.g. `dep`, `glide`). This is not preferred for several reasons:\n  * Some of the other dependency tools (like `glide`) are also unmaintained\n  * go modules are supported by the `go` tool, and have stronger support statements than independent vendoring tools\n  * Defining `go.mod` files for published kubernetes components is desired for interoperability with the go ecosystem\n\n* Move away from vendoring in `k8s.io/kubernetes` as part of the initial move to go modules\n  * To ensure reproducible builds in hermetic build environments based solely on the published repositories, vendoring is still necessary\n  * Moving away from vendoring is orthogonal to moving to go modules and could be investigated/pursued in the future if warranted.\n  * In go1.12.x, vendor-based builds are still the default when building a component located in the GOPATH, so producing components that work when built with go modules or with GOPATH+vendor maximizes interoperability\n\n### Alternatives to publishing staging component modules\n\nSince `require` directives allow locating modules within other modules,\nit is theoretically possible to stop publishing staging component repositories and \nrequire consumers to clone `k8s.io/kubernetes` and reference the staging component\nmodules within that clone.\n\nPros:\n* Removes the need to publish separate staging component repositories\n\nCons:\n* Git SHAs from before 1.15 would not be available as go modules, and would need to continue being accessed via the separately published repositories,\nand [vanity import meta tags](https://golang.org/cmd/go/#hdr-Remote_import_paths) do not allow splitting the location `go get` looks at based on version\n\n* Pointing to a relative location for a module like k8s.io/api works well for references from within kubernetes/kubernetes,\nbut it's unclear how a consumer outside `k8s.io/kubernetes` would use a `replace` directive to let `go build` automatically locate the nested module\n([vanity import meta tags](https://golang.org/cmd/go/#hdr-Remote_import_paths) do not allow anything other than a repository root when pointing at a VCS,\nso we could not indicate \"k8s.io/api is located at k8s.io/kubernetes//staging/src/k8s.io/api\")\n\n* The `k8s.io/kubernetes` repository is extremely large, and forcing a clone of it to pick up `k8s.io/api`, for example, is unpleasant\n\n### Alternative versioning strategies\n\n* switch to tagging major versions on every `kubernetes/kubernetes` release (similar to what client-go does), and use semantic import versioning.\nThis remains a possibility in the future, but requires more tooling and consumer changes to accommodate rewritten imports,\nand doesn't fully allow multiple versions of kubernetes components to coexist as long as there are transitive non-module-based dependencies that change incompatibly over time.\n\n    * consumers\n      * GOPATH consumers\n        * import `k8s.io/apimachinery/...` (as they do today)\n        * are limited to a single copy of kubernetes libraries among all dependencies (as they are today)\n        * `go get` behavior (e.g. `go get client-go`):\n          * uses latest commits of transitive `k8s.io/...` dependencies (likely to break, as today)\n          * uses latest commits of transitive non-module-based dependencies (likely to break, as today)\n      * module-based consumers\n        * reference published modules versions as a consistent `vX.y.z` version (e.g. `v15.0.0`)\n        * import `k8s.io/apimachinery/v15/...` (have to rewrite kubernetes component imports on every major version bump)\n        * can have multiple copies of kubernetes libraries (though non-semantic-import-version transitive dependencies could still conflict)\n        * `go get` behavior (e.g. `GO111MODULE=on go get client-go@v15.0.0`):\n          * uses transitive `k8s.io/{component}/v15/...` dependencies\n          * uses `go.mod`-referenced versions of transitive non-module-based dependencies (unless overridden by top-level module, or conflicting peers referencing later versions)\n    * kubernetes tooling\n      * requires rewriting all `k8s.io/{component}/...` imports to `k8s.io/{component}/vX/...` at the start of each release\n      * requires updating code generation scripts to generate versioned imports for `k8s.io/{api,apimachinery,client-go}`, etc\n    * compatibility implications\n      * allows breaking go changes in each kubernetes \"minor\" release\n      * no breaking go changes are allowed in a kubernetes patch releases (need tooling to enforce this)\n    * allowed versioning changes\n      * modules published this way would have to continue using semantic import versioning\n      * modules published this way could switch to incrementing major/minor versions at a difference cadence as needed\n\n* tag major/minor versions as needed when incompatible changes are made, and use semantic import versioning.\nThis remains a possibility in the future, but requires more tooling and consumer changes to accommodate rewritten imports,\nand doesn't fully allow multiple versions of kubernetes components to coexist as long as there are transitive non-module-based dependencies that change incompatibly over time.\n\n    * consumers\n      * GOPATH consumers\n        * import `k8s.io/apimachinery/...` (as they do today)\n        * are limited to a single copy of kubernetes libraries among all dependencies (as they are today)\n        * `go get` behavior (e.g. `go get client-go`):\n          * uses latest commits of transitive `k8s.io/...` dependencies (likely to break, as today)\n          * uses latest commits of transitive non-module-based dependencies (likely to break, as today)\n      * module-based consumers\n        * reference published modules versions as a variety of `vX.y.z` versions (e.g. `k8s.io/client-go@v15.0.0`, `k8s.io/apimachinery@v15.2.0`, `k8s.io/api@v17.0.0`)\n        * import `k8s.io/apimachinery/v15/...` (have to rewrite kubernetes component imports on every major version bump)\n        * can have multiple copies of kubernetes libraries (though non-semantic-import-version transitive dependencies could still conflict)\n        * `go get` behavior (e.g. `GO111MODULE=on go get client-go@v15.0.0`):\n          * uses transitive `k8s.io/{component}/vX/...` dependencies\n          * uses `go.mod`-referenced versions of transitive non-module-based dependencies (unless overridden by top-level module, or conflicting peers referencing later versions)\n    * kubernetes tooling\n      * requires rewriting all `k8s.io/{component}/vX/...` imports when a major version bump occurs\n      * requires updating code generation scripts to generate versioned imports for `k8s.io/{api,apimachinery,client-go}`, etc\n      * requires tooling to detect when a breaking go change has occurred in a particular component relative to all tagged releases for the current major version\n      * requires tooling to manage versions per component (instead of homogenous versions for staging components)\n    * allowed versioning changes\n      * modules published this way would have to continue using semantic import versioning\n      * modules published this way could switch to incrementing major/minor versions at a difference cadence as needed\n\n\n## Reference\n\n* [@rsc description of options for kubernetes versioning](https://github.com/kubernetes/kubernetes/pull/65683#issuecomment-403705882)\n* `go help modules`\n* https://github.com/golang/go/wiki/Modules, especially:\n  * https://github.com/golang/go/wiki/Modules#semantic-import-versioning\n  * https://github.com/golang/go/wiki/Modules#how-to-prepare-for-a-release\n* https://golang.org/cmd/go/#hdr-The_go_mod_file\n* https://golang.org/cmd/go/#hdr-Maintaining_module_requirements\n* https://golang.org/cmd/go/#hdr-Module_compatibility_and_semantic_versioning\n* [discussion of tagging with v0.x.y](https://github.com/kubernetes/kubernetes/issues/84608)\n"
+  },
+  {
+    "id": "cd1eb4ce83948f70aa9d9379ea918734",
+    "title": "Appropriate use of node-role labels",
+    "authors": ["@smarterclayton"],
+    "owningSig": "sig-architecture",
+    "participatingSigs": ["sig-api-machinery", "sig-network", "sig-node", "sig-testing"],
+    "reviewers": ["@lavalamp", "@derekwaynecarr", "@liggitt"],
+    "approvers": ["@thockin", "@derekwaynecarr"],
+    "editor": "",
+    "creationDate": "2019-07-16",
+    "lastUpdated": "2019-07-16",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Appropriate use of node-role labels\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n- [Proposal](#proposal)\n  - [Use of \u003ccode\u003enode-role.kubernetes.io/*\u003c/code\u003e labels](#use-of--labels)\n  - [Current users of \u003ccode\u003enode-role.kubernetes.io/*\u003c/code\u003e within the project that must change](#current-users-of--within-the-project-that-must-change)\n    - [Service load-balancer](#service-load-balancer)\n    - [Node controller excludes master nodes from consideration for eviction](#node-controller-excludes-master-nodes-from-consideration-for-eviction)\n    - [Kubernetes e2e tests](#kubernetes-e2e-tests)\n    - [Preventing accidental reintroduction](#preventing-accidental-reintroduction)\n- [Design Details](#design-details)\n  - [Migrating existing deployments](#migrating-existing-deployments)\n    - [Instructions for deployers](#instructions-for-deployers)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Future work](#future-work)\n- [Reference](#reference)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n**ACTION REQUIRED:** In order to merge code into a release, there must be an issue in [kubernetes/enhancements] referencing this KEP and targeting a release milestone **before [Enhancement Freeze](https://github.com/kubernetes/sig-release/tree/master/releases) of the targeted release**.\n\nThese checklist items _must_ be updated for the enhancement to be released.\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP: https://github.com/kubernetes/enhancements/issues/1143\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nClarify that the `node-role.kubernetes.io/*` label is for use only by users and external projects and may not be used to vary Kubernetes behavior. Define migration process for all internal consumers of these labels.\n\n## Motivation\n\nThe `node-role.kubernetes.io/master` and the broader `node-role.kubernetes.io` namespace for labels were introduced to provide a simple organizational and grouping convention for cluster users. The labels were reserved solely for organizing nodes via a convention that tools could recognize to display information to end users, and for use by opinionated external tooling that wished to simplify topology concepts. Use of the label by components within the Kubernetes project (those projects subject to API review) was restricted. Specifically, no project could mandate the use of those labels in a conformant distribution, since we anticipated that many deployments of Kubernetes would have more nuanced control-plane topologies than simply \"a control plane node\".\n\nOver time, several changes to Kubernetes core and related projects were introduced that depended on the `node-role.kubernetes.io/master` label to vary their behavior in contravention to the guidance the label was approved under. This was unintentional and due to unclear reviewer guidelines that have since been more strictly enforced. Likewise, the complexity of Kubernetes deployments has increased and the simplistic mapping of control plane concepts to a node has proven to limit the ability of conformant Kubernetes distributions to self-host, as anticipated. The lack of clarity in how to use node-role and the disjoint mechanisms within the code has been a point of confusion for contributors that we wish to remove.\n\nFinally, we wish to clarify that external components may use node-role tolerations and labels as they wish as long as they are cognizant that not all conformant distributions will expose or allow those tolerations or labels to be set.\n\n\n### Goals\n\nThis KEP:\n\n* Clarifies that the use of the `node-role.kubernetes/*` label namespace is reserved solely for end-user and external Kubernetes consumers, and:\n  * Must not be used to vary behavior within Kubernetes projects that are subject to API review (kubernetes/kubernetes and all components that expose APIs under the `*.k8s.io` namespace)\n  * Must not be required to be present for a cluster to be conformant\n* Describes the locations within Kubernetes that must be changed to use an alternative mechanism for behavior\n  * Suggests approaches for each location to migrate\n* Describes the timeframe and migration process for Kubernetes distributions and deployments to update labels\n\n\n## Proposal\n\n### Use of `node-role.kubernetes.io/*` labels\n\n* Kubernetes components MUST NOT set or alter behavior on any label within the `node-role.kubernetes.io/*` namespace.\n* Kubernetes components (such as `kubectl`) MAY simplify the display of `node-role.kubernetes.io/*` labels to convey the node roles of a node\n* Kubernetes examples and documentation MUST NOT leverage the node-role labels for node placement\n* External users, administrators, conformant Kubernetes distributions, and extensions MAY use `node-role.kubernetes.io/*` without reservation\n  * Extensions are recommended not to vary behavior based on node-role, but MAY do so as they wish\n* First party components like `kubeadm` MAY use node-roles to simplify their own deployment mechanisms.\n* Conformance tests MUST NOT depend on the node-role labels in any fashion\n* Ecosystem controllers that desire to be placed on the masters MAY tolerate the node-role master taint or set nodeSelector to the master nodes in order to be placed, but SHOULD recognize that some deployment models will not have these node-roles, or may prohibit deployments that attempt to schedule to masters as unprivileged users. In general we recommend limiting this sort of placement rule to examples, docs, or simple deployment configurations rather than embedding the logic in code.\n\n\n### Current users of `node-role.kubernetes.io/*` within the project that must change\n\nThe following components vary behavior based on the presence of the node-role labels:\n\n\n#### Service load-balancer\n\nThe service load balancer implementation previously implemented a heuristic where `node-role.kubernetes.io/master` is used to exclude masters from the candidate nodes for a service. This is an implementation detail of the cluster and is not allowed. Since there is value in excluding nodes from service load balancer candidacy in some deployments, an alpha feature gated label `alpha.service-controller.kubernetes.io/exclude-balancer` was added in Kubernetes 1.9.\n\nThis label should be moved to beta in Kube 1.17 at its final name `node.kubernetes.io/exclude-from-external-load-balancers`, its feature gate `ServiceNodeExclusion` should default on in 1.18, the gate `ServiceNodeExclusion` should be declared GA in 1.19, and the gate will be removed in 1.20. The old alpha label should be honored in 1.17 and removed in 1.18.\n\nStarting in 1.16 the legacy code block should be gated on `LegacyNodeRoleBehavior=true`\n\n\n#### Node controller excludes master nodes from consideration for eviction\n\nThe `k8s.io/kubernetes/pkg/util/system/IsMasterNode(nodeName)` function is used by the NodeLifecycleController to exclude nodes with a node name that ends in `master` or starts with `master-` when considering whether to mark nodes as disrupted. A recent PR attempted to change this to use node-roles and was blocked. Instead, the controller should be updated to use a label `node.kubernetes.io/exclude-disruption` to decide whether to exclude nodes from being considered for disruption handling.\n\n\n#### Kubernetes e2e tests\n\nThe e2e tests use a number of heuristics including the `IsMasterNode(nodeName)` function and the node-roles labels to select nodes. In order for conformant Kubernetes clusters to run the tests, the e2e suite must change to use individual user-provided label selectors to identify nodes to test, nodes that have special rules for testing unusual cases, and for other selection behaviors. The label selectors may be defaulted by the test code to their current values, as long as a conformant cluster operator can execute the e2e suite against an arbitrary cluster.\n\nThe `IsMasterNode()` method will be moved to be test specific, identified as deprecated, and will be removed as soon as possible.\n\nQUESTION: Is a single label selector sufficient to identify nodes to test?\n\n\n#### Preventing accidental reintroduction\n\nIn order to prevent reviewers from accidentally allowing code changes that leverage this functionality, we should clarify the Godoc of the constant to limit their use.  A lint process could be run as part of verify that requires approval of a small list to modify exclusions (currently only cmd/kubeadm will be allowed to use that constaint, with all test function being abstracted). The review doc should call out that labels must be scoped to a particular feature enablement vs being broad.\n\nSome components like the external cloud provider controllers (considered to fall within these rules due to implementing k8s.io APIs) may be vulnerable to accidental assumptions about topology - code review and e2e tests are our primary mechanism to prevent regression.\n\n\n## Design Details\n\n### Migrating existing deployments\n\nThe proposed fixes will all require deployment-level changes. That must be staged across several releases, and it should be possible for deployers to move early and \"fix\" the issues that may be caused by their topology.\n\nTherefore, for each change we recommend the following process to adopt the new labels in successive releases:\n\n* Release 1 (1.16):\n  * Introduce a feature gate for disabling node-role being honored. The gate defaults to on. `LegacyNodeRoleBehavior=true`\n  * Define the new node label with an associated feature gate for each feature area. The gate defaults to off. `ServiceNodeExclusion=false` and `NodeDisruptionExclusion=false`\n  * Behavior for each functional area is defined as `(LegacyNodeRoleBehavior == on \u0026\u0026 node_has_role) || (FeatureGate == on \u0026\u0026 node_has_label)`\n  * No new components may leverage node-roles within Kubernetes projects.\n  * Early adopters may label their nodes to opt in to the features, even in the absence of the gate.\n* Release 2 (1.17):\n  * The legacy alpha label `alpha.service-controller.kubernetes.io/exclude-balancer` is marked as deprecated\n  * Deprecation of node role behavior in tree is announced for 1.20, with a detailed plan for cluster administrators and deployers\n  * Gates are officially alpha\n* Release 3 (1.18):\n  * The old label `alpha.service-controller.kubernetes.io/exclude-balancer` is removed\n  * For both labels, usage is reviewed and as appropriate the label is declared beta/GA and the feature gate is set on\n  * All Kubernetes deployments should be updated to add node labels as appropriate: `kubectl label nodes -l node-role.kubernetes.io/master LABEL_A=VALUE_A`\n  * Documentation will be provided on making the transition\n  * Deployments may set `LegacyNodeRoleBehavior=false` after they have set the appropriate labels.\n* Release 4 (1.19):\n  * Default the legacy gate `LegacyNodeRoleBehavior` to off. Admins whose deployments still use the old labels may set `LegacyNodeRoleBehavior=true` during 1.19 to get the legacy behavior.\n  * Deployments should stop setting `LegacyNodeRoleBehavior=false` if they opted out early.\n* Release 5 (1.20):\n  * The `LegacyNodeRoleBehavior` gate and all feature-level gates are removed, components that attempt to set these gates will fail to start.\n  * Code that references node-roles within Kubernetes will be removed.\n\nIn Release 5 (which could be as early as 1.20) this KEP will be considered complete.\n\n#### Instructions for deployers\n\nThe current behavior of the `node-role.kubernetes.io/master` label on nodes preventing them from being part of service load balancers or from being disrupted when NotReady is deprecated and will be fully removed in Kubernetes 1.20. Administrators and Kubernetes deployers should follow these steps.\n\nIf you are using the `alpha.service-controller.kubernetes.io/exclude-balancer` label in your deployments to exclude specific nodes from your deployment, the label has been replaced in 1.17 with `node.kubernetes.io/exclude-from-external-load-balancers`.  All administrators should run the following command before upgrading to Kubernetes 1.18 and set the feature gate `ServiceNodeExclusion=true`:\n\n    kubectl label nodes --selector=alpha.service-controller.kubernetes.io/exclude-balancer \\\n        node.kubernetes.io/exclude-balancer=true\n\nCluster deployers that rely on the existing behavior where master nodes are not part of the service load balancer and master workloads will not be evicted if the master is NotReady for longer than the grace period should run the following command after upgrading to Kubernetes 1.18:\n\n    kubectl label nodes --selector=node-role.kubernetes.io/master \\\n        node.kubernetes.io/exclude-from-external-load-balancers=true \\\n        node.kubernetes.io/exclude-disruption=true\n\nAfter setting these labels in 1.18, administrators will need to take no further action.\n\nCluster deployers that wish to manage this migration during the 1.17 to 1.18 upgrade should label nodes and set feature gates before upgrading to 1.18. If `LegacyNodeRoleBehavior=false` is set, it must be removed prior to the 1.19 to 1.20 upgrade.\n\n\n### Test Plan\n\n* Unit tests to verify selection using feature gates\n\n### Graduation Criteria\n\n* New labels and feature flags become beta after one release, GA and defaulted on after two, and are removed after two releases after they are defaulted on (so 4 releases from when this is first implemented).\n* Documentation for migrating to the new labels is available in 1.18.\n\n### Upgrade / Downgrade Strategy\n\nAs described in the migration process, deployers and administrators have 2 releases to migrate their clusters.\n\n### Version Skew Strategy\n\nControllers are updated after the control plane, so consumers must update the labels on their nodes before they update controller processes in 1.19.\n\n## Implementation History\n\n- 2019-07-16: Created\n\n## Future work\n\nThis proposal touches on the important topic of scheduling policy - the ability of clusters to restrict where arbitrary workloads may run - by noting that some conformant clusters may reject attempts to schedule onto masters. This is out of scope of this KEP except to indicate that node-role use by ecosystem components may conflict with future enhancements in this area.\n\n\n## Reference\n\n* https://groups.google.com/d/msg/kubernetes-sig-architecture/ZKUOPy2PNJ4/lDh4hs4HBQAJ\n* https://github.com/kubernetes/kubernetes/pull/35975\n* https://github.com/kubernetes/kubernetes/pull/39112\n* https://github.com/kubernetes/kubernetes/pull/76654\n* https://github.com/kubernetes/kubernetes/pull/80021\n* https://github.com/kubernetes/kubernetes/pull/78500 - Work to remove master role label from e2e\n"
+  },
+  {
+    "id": "baef4f38149a2e1741187eb902e78f28",
+    "title": "Behavior-driven Conformance Testing",
+    "authors": ["@johnbelamaric", "@hh"],
+    "owningSig": "sig-architecture",
+    "participatingSigs": ["sig-testing"],
+    "reviewers": ["@timothysc", "@spiffxp", "@alejandrox1", "@johnschnake"],
+    "approvers": ["@bgrant0607", "@smarterclayton"],
+    "editor": "TBD",
+    "creationDate": "2019-04-12",
+    "lastUpdated": "2010-07-30",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Behavior-driven Conformance Testing\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Representation of Behaviors](#representation-of-behaviors)\n  - [Behavior and Test Generation Tooling](#behavior-and-test-generation-tooling)\n    - [Handwritten Behaviour Scenarios](#handwritten-behaviour-scenarios)\n  - [Coverage Tooling](#coverage-tooling)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Phase 1](#phase-1)\n    - [Tying tests back to behaviors](#tying-tests-back-to-behaviors)\n    - [kubetestgen](#kubetestgen)\n  - [Phase 2](#phase-2)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Future development](#future-development)\n    - [Complex Storytelling combined with json/yaml](#complex-storytelling-combined-with-jsonyaml)\n    - [Example patch test scenario](#example-patch-test-scenario)\n    - [Generating scaffolding from Gherkin .feature files](#generating-scaffolding-from-gherkin-feature-files)\n    - [Autogeneration of Test Scaffolding](#autogeneration-of-test-scaffolding)\n    - [Combining gherkin with existing framework](#combining-gherkin-with-existing-framework)\n- [Implementation History](#implementation-history)\n- [Drawbacks](#drawbacks)\n- [Alternatives](#alternatives)\n  - [Annotate test files with behaviors](#annotate-test-files-with-behaviors)\n  - [Annotate existing API documentation with behaviors](#annotate-existing-api-documentation-with-behaviors)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n**ACTION REQUIRED:** In order to merge code into a release, there must be an issue in [kubernetes/enhancements] referencing this KEP and targeting a release milestone **before [Enhancement Freeze](https://github.com/kubernetes/sig-release/tree/master/releases)\nof the targeted release**.\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e., [kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be updated for the enhancement to be released.\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [x] KEP approvers have set the KEP status to `implementable`\n- [x] Design details are appropriately documented\n- [x] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [x] Graduation criteria is in place\n- [x] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nThis proposal modifies the conformance testing framework to be driven by a list\nof agreed upon behaviors. These behaviors are identified by processing of the\nAPI schemas, documentation, expert knowledge, and code examination. They are\nexplicitly documented and tracked in the repository rather than in GitHub\nissues, allowing them to be reviewed and approved independently of the tests\nthat evaluate them. Additionally it proposes new tooling to generate tests, test\nscaffolding, and test coverage reports.\n\n## Motivation\n\nIt has proven difficult to measure how much of the expected Kubernetes behavior\nthe current conformance tests cover. The current measurements are based upon\nidentifying which API endpoints are exercised by the tests. The Kubernetes API\nis CRUD-oriented, and most of the client’s desired behavior is encapsulated in\nthe payload of the create or update calls, not in the simple fact that those\nendpoints were hit. This means that even if a given endpoint is shown as\ncovered, it’s impossible to know how much that tests the actual behavior.\n\nCoverage is measured this way because there is no single, explicit list of\nbehaviors that comprise the expected behavior of a conformant cluster. These\nexpectations are spread out across the existing design documents, KEPs, the user\ndocumentation, a subset of the e2e test, and the code itself. This makes it\nimpossible to identify if the conformance suite provides a meaningful test of a\ncluster’s operation.\n\nAdditionally, progress in writing and promoting tests has been slow and too much\nmanual effort is involved. As a starting point, this proposal includes new\ntooling that uses the API schemas to identify expected behaviors and produce\ntests and test scaffolding to quickly cover those behaviors.\n\n### Goals\n\n* Enable separate review of behaviors and tests that evaluate those behaviors.\n* Provide a single location for defining conforming behavior.\n* Provide tooling to generate as many of the behaviors as possible from API\n  schemas. This will be a seed for the behavior lists, which will in turn be\n  human curated. Refinements can improve the quality of the seed over time.\n* Provide tooling to generate tests and test scaffolding for validating\n  behaviors.\n* Provide tooling to measure the conformance test coverage of the behaviors.\n* Provide an incremental migration path from current conformance testing to the\n  updated framework.\n\n### Non-Goals\n\n* Develop a complete set of behaviors that define a conforming cluster. This is\n  an ongoing effort, and this proposal is intended to make that more efficient.\n* Add new conformance tests. It is expected that during this effort new\n  tests may be created using the proposed tooling, but it is not explicitly part\n  of this proposal.\n* Provide tooling that perfectly populates the behaviors from the API schema.\n  Not enough information is present in the schema to achieve this. The tooling\n  is only intended to produce a seed for human curation.\n\n## Proposal\n\nThe proposal consists of four deliverables:\n* A machine readable format to define conforming behaviors.\n* Tooling to generate lists of behaviors from the API schemas.\n* Tooling to generate tests and test scaffolding to evaluate those behaviors.\n* Tooling to compare the implemented tests to the list of behaviors and\n  calculate coverage.\n\n### Representation of Behaviors\n\nBehaviors will be captured in prose, which is in turn embedded in a YAML file\nalong with meta-data about the behavior.\n\nBehaviors must be captured in the repository and agreed upon as required for\nconformance. Behaviors are broken into feature areas, and there are multiple\ntest suites (named sets of tests) for each feature area. Some of these suites\nmay be machine-generated based upon the API schema, whereas others are\nhandwritten. Keeping the generated and handwritten suites in separate files\nallows regeneration of the auto-discovered behavior suites. Some areas may be\ndefined by API group and Kind, while others will be subjectively defined by\nsubject-matter experts.\n\nValidation and conformance designations are made on a per-suite basis,\nnot a per-behavior basis. There may be multiple suites in a feature area\nthat are required for validation and/or conformance.\n\nThe grouping at the suite level should be defined based upon subjective\njudgement of how behaviors relate to one another, along with an understanding\nthat all behaviors in a given suite may be required to function for a given\ncluster to pass validation for that suite.\n\nTypical suites defined for any given feature will include:\n * API spec. This suite is generated from the API schema and represents\n   the basic field-by-field functionality of the feature. For features\n   that include provider-specific fields (for example, various VolumeSource\n   fields for pods), those must be segregated into separate suites.\n * Internal interactions. This suite tests interactions between settings\n   of fields within the API schema for this feature.\n * External interactions. This suite tests interactions between this feature\n   and other features.\n\nEach suite may be stored in a separate file in a directory for the specific\narea. For example, a \"Pods\" area would be structured as a `pods` directory with\nthese files:\n * `api-generated.yaml` describing the set of behaviors auto-generated from the\n   API specification.\n * `lifecycle.yaml` describing the set of behaviors expected from the Pod\n   lifecycle.\n\nBehavior files are reviewed separately from the tests themselves, with separate\nOWNERs files corresponding to those tests. This may be captured in a directory\nstructure such as:\n\n```\ntest/conformance\n├── behaviors\n│   ├── OWNERS # no-parent: true, approvers: behavior-approvers\n│   └── {area}\n│       ├── OWNERS # optional: reviewers: area-experts\n│       └── {suite}.yaml\n├── OWNERS # approvers: test-approvers\n└── tests.yaml # promotion updates this file; tests MUST map to a behavior\n```\n\nThe structure of the behavior YAML files is described by these Go types:\n\n```go\n// Area defines a general grouping of behaviors\ntype Area struct {\n        // Area is the name of the area.\n        Area   string  `json:\"area,omitempty\"`\n\n        // Suites is a list containing each suite of behaviors for this area.\n        Suites []Suite `json:\"suites,omitempty\"`\n}\n\ntype Suite struct {\n        // Suite is the name of this suite.\n        Suite       string     `json:\"suite,omitempty\"`\n\n        // Level is `Conformance` or `Validation`.\n        Level       string     `json:\"level,omitempty\"`\n\n        // Description is a human-friendly description of this suite, possibly\n        // for inclusion in the conformance reports.\n        Description string     `json:\"description,omitempty\"`\n\n        // Behaviors is the list of specific behaviors that are part of this\n        // suite.\n        Behaviors   []Behavior `json:\"behaviors,omitempty\"`\n}\n\ntype Behavior struct {\n        // Id is a unique identifier for this behavior, and will be used to tie\n        // tests and their results back to this behavior. For example, a\n        // behavior describing the defaulting of the PodSpec nodeSelector might\n        // have an id like `pods/spec/nodeSelector/default`.\n        Id          string `json:\"id,omitempty\"`\n\n        // ApiObject is the object whose behavior is being described. In\n        // particular, in generated behaviors, this is the object to which\n        // ApiField belongs. For example, `core.v1.PodSpec` or\n        // `core.v1.EnvFromSource`.\n        ApiObject   string `json:\"apiObject,omitempty\"`\n\n        // ApiField is filled out for generated tests that are testing the\n        // behavior associated with a particular field. For example, if\n        // ApiObject is `core.v1.PodSpec`, this could be `nodeSelector`.\n        ApiField    string `json:\"apiField,omitempty\"`\n\n        // ApiType is the data type of the field; for example, `string`.\n        ApiType     string `json:\"apiType,omitempty\"`\n\n        // Generated is set to `true` if this entry was generated by tooling\n        // rather than hand-written.\n        Generated   bool   `json:\"generated,omitempty\"`\n\n        // Description specifies the behavior. For those generated from fields,\n        // this will identify if the behavior in question is for defaulting,\n        // setting at creation time, or updating, along with the API schema field\n        // description.\n        Description string `json:\"description,omitempty\"`\n}\n```\n\n### Behavior and Test Generation Tooling\n\nSome sets of behaviors may be tested in a similar, mechanical way. Basic CRUD\noperations, including updates to specific fields and constraints on immutable\nfields, operate in a similar manner across all API resources. Given this, it is\nfeasible to automate the creation of simple tests for these behaviors, along\nwith the behavior descriptions in the `api-generated.yaml`. In some cases a\ncomplete test may not be easy to generate, but a skeleton may be created that\ncan be converted into a valid test with minimal effort.\n\nFor these tests, the input is a set of manifests that are applied to the\ncluster, along with a set of conditions that are expected to be realized within\na specified timeframe. The test framework will apply the manifests, and monitor\nthe cluster for the conditions to occur; if they do not occur within the\ntimeframe, the test will fail.\n\nFor each Spec object, scaffolding can be defined to include the following tests:\n\n* Creation and read of the resource with only required fields specified.\n  * API functions as expected: Resource is created and may be read, and defaults\n    are set. This is mechanical and can be completely generated.\n  * Cluster behaves as expected. This cannot be generated, but a skeleton can be\n    generated that allows test authors to evaluate the condition of the cluster\n    to make sure it meets the expectations.\n* Deletion of the resource. This may be mostly mechanical but if there are side-\n  effects, such as garbage collection of related resources, we may want to have\n  manually written evaluation here as well.\n* Creation of resource with each field set, and update of each mutable field.\n  * For each mutable field, apply a patch to the based resource definition\n    before creation (for create tests), or after creation (for update tests).\n  * Evaluate that the API functions as expected; this is mechanical and\n    generated.\n  * Evaluate that the cluster behaves as expected. In some cases this may be\n    able to re-use the same evaluation function used during the creation tests,\n    but often it will require hand-crafted code to test the conditions.\n    Nonetheless, the scaffolding can be generated, minimizing the effort needed\n    to implement the test.\n\nAs an example, the tooling would generate scaffolding which creates a Pod. It\nwould still be necessary to fill in the values used for the base Pod fixture.\nThe tooling would also generate a test case that includes a change to `image:`\nfield of the container spec. It would still be necessary for a human to fill in\nthe what new value to use for the image. The scaffolding would also generate the\nentire test evaluation as described, except that the true/false condition for\nwhether the desired state is achieved would be an empty function that needs to\nbe implemented. In this case, the function would wait for the condition that the\nPod's container has been restarted with the new image.  While there is still\nhuman involvement here, much of the toil is removed, with the only necessary\nintervention being specifying the specific image values and content of the\nfunction.\n\nThis example does illustrate the need for some logic in the generation to avoid\noverwriting the specified image values and function content. One option is to\nput the fixtures in separate files, rather than embedding them in the generated\nfiles. However that extra indirection and all the extra files can make the tests\ndifficult to follow. Some more investigation is necessary here.\n\n#### Handwritten Behaviour Scenarios\n\nAdditional, handwritten tests will be needed that modify the resource in\nmultiple ways and evaulate the behavior. The scaffolding must be built such that\nthe same process is used for these tests. The test author must only need to\ndefine:\n* A patch to apply to create or update the resource.\n* A function to evaluate the effect of the API call.\n\nWith those two, the same creation and update scaffolding defined for individual\nfield updates can be reused.\n\n### Coverage Tooling\n\nIn order to tie behaviors back to the tests that are generated, including\nexisting e2e tests that already cover behaviors, new tags with behavior IDs will\nbe added to conformance tests. Using the existing conformance framework\nmechanism allows incremental adoption of this proposal. Thus, rather than a new\nconformance framework function, test authors will indicate the behaviors covered\nby their tests with a tag in the `framework.ConformanceIt` call.\n\nThis also enables a single test to validate multiple behaviors, although that\nshould be discouraged.\n\n### Risks and Mitigations\n\nThe behavior definitions may not be properly updated if a change is made to a\nfeature, since these changes are made in very different areas in the code.\nHowever, given that the behaviors defining conformance are generally stable,\nthis is not a high risk.\n\n## Design Details\n\nDelivery of this KEP shall be done in the following phases:\n\n### Phase 1\n\nIn Phase 1, we will:\n* Implement the behavior formats and types described above. This will include\n  separate suites for tooling-generated behaviors and handcrafted behaviors.\n* Implement the directory structure described above to contain the behavior\n  lists, including how to tie tests back to behaviors.\n* `kubetestgen`, a tool which reads the OpenAPI schema and generates the list of\n  behaviors.\n* Migrate existing conformance tests to work with the new tooling. Existing\n  tooling around generation of conformance reports will not be changed in this\n  phase.\n\n#### Tying tests back to behaviors\nThe proposal above mentions `tests.yaml` but does not describe a format for that\nfile. The current conformance frameworks requests that during promotion of the\ntest to conformance, the developer adds metadata, including the release name,\nthe test name, and description. Tests are identified in the\n[conformance.txt](https://github.com/kubernetes/kubernetes/blob/master/test/conformance/testdata/conformance.txt)\nfile by their Ginko description. Unfortunately, this does not produce unique\ntest names, as it does not include all of the `Describe` calls from higher in\nthe call tree (see this\n[slack discussion](https://kubernetes.slack.com/archives/C78F00H99/p1566324743171500)\nfor more details).\n\nAs part of this KEP, tests being promoted to conformance must add a unique\nidentifier, `TestId`, in their conformance metadata. This will be used, along with\nthe behavior IDs, to map which tests validate which behaviors in the\n`tests.yaml` file. The Go structures for `tests.yaml` are shown below.\n\n```go\ntype BehaviorTestList struct {\n       Tests []BehaviorTest `json:\"tests,omitempty\"`\n}\n\ntype BehaviorTest struct {\n        BehaviorId  string `json:\"behaviorId,omitempty\"`\n        TestId      string `json:\"testId,omitempty\"`\n\n        // Description is optional and is intended to make reviewing easier; the\n        // expectation would be that tooling would copy the value here.\n        Description string `json:\"description,omitempty\"`\n}\n```\n\n#### kubetestgen\n\nIn this phase, the tool will only generate behavior lists in the format defined\nabove. It will accept the following flags:\n* `-schema` - a URL or local file name pointing to the JSON OpenAPI schema\n* `-resource` - the specific OpenAPI definition for which to generate behaviors\n* `-area` - the name to use for the area\n* `-suite` - the name to use for the suite\n* `-behaviorsdir` - the path to the behaviors directory (default current\n  directory)\n\nThe tool will read the schema, locate the specific definition, and generate the\n`{area}` directory and `{suite}.yaml` as described in the proposal above.\n\n### Phase 2\n\nIn Phase 2, we will:\n* Migrate existing tooling for conformance report generation to the new method,\n  and remove older tooling. This will eliminate the need to maintain conformance\n  tests in both the new and old manner.\n* Add test scaffolding generation in parallel with the behavior list generation.\n* Implement coverage metrics comparing behavior lists to the coverage captured\n  by existing conformance tests.\n\n### Graduation Criteria\nAs this is a tooling component and is not user facing, it does not follow the\nordinary alpha/beta/GA process. In 1.17, the intent is to implement Phase 1,\nwithout disruption to any feature development. The acceptance criteria here\nare that the deliverables described in Phase 1 are complete, and that no\ndevelopers other than those writing or promoting conformance tests are\naffected by the changes introduced in this KEP.\n\n### Future development\n\nThe description above achieves the basic goals of the KEP. However, in the same\ntimeframe as implementation of this KEP, we also plan to explore some future\nrefinements. In particular, we will explore the use of an existing behavior-\ndriven testing language to refine our *prose* behavior descriptions into\n*machine-readable* behavior descriptions.\n\nOne such language is [Gherkin](https://cucumber.io/docs/gherkin/). In Gherkin,\nspecifications are defined around Features, which are collections of Scenarios.\n\n#### Complex Storytelling combined with json/yaml\n\nInline json or yaml as CRUD input/output can be autogenerated for verification. The\njson or yaml can also be contained in external files. The functions matching the\nstep definitions would be re-used for all matching scenarios as needed.\n\n```feature\nFeature: Intrapod Communication\n  Pods need to be able to talk to each other, as well as the node talking to the Pod.\n  @sig-node @sig-pod\n  Scenario: Nodes can communicate to each other\n    Given a pods A and B\n    When pod A says hello to pod B\n    Then pod B says hello to pod A\n  @wip @tags-are-no-longer-part-of-test-names\n  Scenario: Pods can can communicate to Nodes\n    Given a pod A on a node\n    When the node says hello to pod A\n    Then pod A says hello to the node\n    And this is fine\n```\n\n#### Example patch test scenario\n\n```feature\nFeature: Manually using Manifests to CRUD and evaluate effects\n  Pods need to be able to talk to each other, as well as the node talking to the Pod.\n  Scenario: Pods can can communicate to Nodes\n    Given I create pod A with this yaml spec\n      \"\"\"\n      yaml: [\n         values\n      ]\n      \"\"\"\n    And I create pod B with this json spec\n      \"\"\"\n      {\n        json: values\n      }\n      \"\"\"\n    When I request pod A and pod B talk to each other\n    Then I can observe a v1.PodCommunication matching this json spec\n      \"\"\"\n      {\n        \"node a\": \"talked to node b\"\n      }\n      \"\"\"\n    And this is fine\n```\n#### Generating scaffolding from Gherkin .feature files\n\nA Gherkin **feature** is synonymous with our definition of **behaviour**, and\ntagging can be used for **@conformance** or **@release-X.Y** metadata.\n\n```feature\nFeature: Structured Metadata allowing Behaviour Driven tooling automation\n  In order to auto-generate testing scaffolding\n  As a sig-X member\n  I want to describe the behaviour of X\n\n  @sig-X\n  Scenario: Behaviour X\n    Given a well formed file describing the behaviour X\n    When I run the automation\n    Then I am provided with the basic structure for a corresponding test\n    And this is fine\n  @sig-Y\n  Scenario: Behaviour Y\n    Given a well formed file describing the behaviour Y\n    When I run the automation\n    Then I am provided with the basic structure for a corresponding test\n    And this is fine\n  @sig-Y @sig-X\n  Scenario: Behaviour X+Y\n    Given a well formed file describing the behaviour X\n    And a well formed file describing the behaviour Y\n    When I run the automation\n    Then I can reuse existing step definitons on multiple tests\n    And this is fine\n```\n\n#### Autogeneration of Test Scaffolding\n\n```shell\n~/go/bin/godog --no-colors\n```\n\n```feature\nFeature: Structured Metadata allowing Behaviour Driven tooling automation\n  In order to auto-generate testing scaffolding\n  As a sig-X member\n  I want to describe the behaviour of X\n\n  Scenario: Behaviour X                                                  # features/behaviour.feature:7\n    Given a well formed file describing the behaviour X\n    When I run the automation\n    Then I am provided with the basic structure for a corresponding test\n    And this is fine\n\n  Scenario: Behaviour Y                                                  # features/behaviour.feature:13\n    Given a well formed file describing the behaviour Y\n    When I run the automation\n    Then I am provided with the basic structure for a corresponding test\n    And this is fine\n\n  Scenario: Behaviour X+Y                                       # features/behaviour.feature:19\n    Given a well formed file describing the behaviour X\n    And a well formed file describing the behaviour Y\n    When I run the automation\n    Then I can reuse existing step definitons on multiple tests\n    And this is fine\n\n3 scenarios (3 undefined)\n13 steps (13 undefined)\n1.253405ms\n\nYou can implement step definitions for undefined steps with these snippets:\n\nfunc aWellFormedFileDescribingTheBehaviourX() error {\n  return godog.ErrPending\n}\n\nfunc iRunTheAutomation() error {\n  return godog.ErrPending\n}\n\nfunc iAmProvidedWithTheBasicStructureForACorrespondingTest() error {\n  return godog.ErrPending\n}\n\nfunc thisIsFine() error {\n  return godog.ErrPending\n}\n\nfunc aWellFormedFileDescribingTheBehaviourY() error {\n  return godog.ErrPending\n}\n\nfunc iCanReuseExistingStepDefinitonsOnMultipleTests() error {\n  return godog.ErrPending\n}\n\nfunc FeatureContext(s *godog.Suite) {\n  s.Step(`^a well formed file describing the behaviour X$`, aWellFormedFileDescribingTheBehaviourX)\n  s.Step(`^I run the automation$`, iRunTheAutomation)\n  s.Step(`^I am provided with the basic structure for a corresponding test$`, iAmProvidedWithTheBasicStructureForACorrespondingTest)\n  s.Step(`^this is fine$`, thisIsFine)\n  s.Step(`^a well formed file describing the behaviour Y$`, aWellFormedFileDescribingTheBehaviourY)\n  s.Step(`^I can reuse existing step definitons on multiple tests$`, iCanReuseExistingStepDefinitonsOnMultipleTests)\n}\n\n```\n\nThese functions and the Suite.Step matchers that tie them to Gherkin steps can\nbe pasted into a `test_steps.go` file as a initial scaffolding.\n\n#### Combining gherkin with existing framework\n\nOur current tests are not super easy to write, read, or review. BDD in go was in\nit's early days when k8s started integration testing with a closely coupled\ncomponent testing approach. Our Ginko based e2e framework evolved based upon\nthose tightly coupled assumptions. This approach unfortunately lacks the\nmetadata, tags, and descriptions of the desired behaviours required for clear\nseparation of acceptance behaviors and tests.\n\nDocumenting and discovering of all our behaviours will require a combination of\nautomated introspection and well as some old fashioned human storytelling.\n\nTo do so need to standardize the business language that our bottlenecked people\ncan use to write these stories in a way can be assisted with some automation.\nThis would reduce complexity for articulating concrete requirements for\nexecution in editors, humans, and automation workflows.\n\nDefining our Behaviours in Gherkin would allow us to leverage our existing\nconformance framework and test mechanisms to allow incremental adoption of this\nproposal.\n\nScenarios could be defined for existing tests using the form:\n\n```feature\nScenario: Use existing ginkgo framework\n  As a test contributor\n  I want to not throw away all our old tests\n  In order to retain the value generated in them\n  @sig-node @sig-pod @conformance @release-1.15\n  Feature: Map behaviours to existing ginkgo tests\n    Given existing test It('should do the right thing')\n    And I optionally tag it with @conformance\n    When I run the test\n    Then we utilize our existing test via our new .feature framework\n    And this is fine\n```\n\nThus, test authors will indicate the behaviors covered by adding a\n**@conformance** tag to Feature/Behaviours using `Given an existing test\nIt('test string')`\n\n\n## Implementation History\n\n- 2019-04-12: Created\n- 2019-06-11: Updated to include behavior and test generating from APIs.\n- 2019-07-08: Updated to include Gherkin / godog as possible behaviour workflow\n- 2019-07-24: Updated to add reviewers and an example on generated scaffolding\n- 2019-07-30: Updated to separate Gherkin / godog into second phase, include\n  directory structure for showing behavior/test separation\n- 2019-10-01: Added detailed design; marked implementable\n\n## Drawbacks\n\n* Separating behaviors into a file that is not directly part of the test suite\n  creates an additional step for developers and could lead to divergence.\n\n## Alternatives\n\n### Annotate test files with behaviors\n\nThis option is essentially an extension of the existing tagging of e2e tests.\nRather than just tagging existing tests, we can embed the list of behaviors in\nthe files as well. The same set of metadata that is described in Option 1 can be\nembedded as specialized directives in comments.\n\n\n*Pros*\n* Keeps behaviors and tests together in the same file.\n\n*Cons*\n* All of the same features may be met, but the tooling needs to parse the Go\n  code and comments, which is more difficult than parsing a YAML.\n* Behaviors are scattered throughout test files and intermingled with test code,\n  making it hard to review whether the list of behaviors is complete (this\n  could be mitigated with tooling similar to the existing tooling that extracts\n  test names).\n* Adding or modifying desired behaviors requires modifying the test files, and\n  leaving the behaviors with a TODO or similar flag for tracking what tests are\n  needed.\n\n### Annotate existing API documentation with behaviors\nThe current API reference contains information about the meaning and expected\nbehavior of each API field. Rather than producing a separate list, the metadata\nfor conformance tests can be attached to that documentation.\n\n*Pros*\n* Avoids adding a new set of files that describe the behavior, leveraging what\n  we already have.\n* API reference docs are a well-known and natural place to look for how the\n  product should behave.\n* It is clear if a given API or field is covered, since it is annotated directly\n  with the API.\n\n*Cons*\n* Behaviors are spread throughout the documentation rather than centrally\n  located.\n* It may be difficult to add tests that do not correspond to specific API\n  fields.\n"
+  },
+  {
+    "id": "aee8bbaad5331a1e376c497af3894dc2",
+    "title": "Production Readiness Review Process",
+    "authors": ["@johnbelamaric"],
+    "owningSig": "sig-architecture",
+    "participatingSigs": ["sig-release"],
+    "reviewers": ["@derekwaynecarr", "@vishh", "@justaugustus", "@alejandrox1"],
+    "approvers": ["@derekwaynecarr", "@dims"],
+    "editor": "",
+    "creationDate": "2019-07-31",
+    "lastUpdated": "2019-11-12",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Production Readiness Review Process\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n**ACTION REQUIRED:** In order to merge code into a release, there must be an issue in [kubernetes/enhancements] referencing this KEP and targeting a release milestone **before [Enhancement Freeze](https://github.com/kubernetes/sig-release/tree/master/releases)\nof the targeted release**.\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e., [kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be updated for the enhancement to be released.\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nThis proposal introduces a production readiness review process for features\nmerging as alpha or graduating to beta or GA. The review process is intended to\nensure that features merging into Kubernetes are observable and supportable,\ncan be safely operated in production environments, and can be disabled or rolled\nback in the event they cause increased failures in production.\n\n## Motivation\n\nKubernetes has grown quickly and organically. The KEP process introduced a\nmechanism to help build consensus and ensure the community supports the\ndirection and implementation of various features. It provides a way to document\nrelease and graduation criteria, but it leaves that criteria up to the KEP\nauthors and approvers to define. Because these are normally the developers and\nthe SIGs associated with the KEP, there is not always a clear representation of\nthe view of cluster operators. This can result in operational and supportability\noversights in the graduation criteria.\n\nThis KEP proposes a process to ensure that production concerns are addressed in\nall new features, at a level appropriate to the features' maturity levels.\n\n* TBD: Document some experiences that motivate the need for this.\n\n### Goals\n\n* Define production readiness criteria for alpha, beta, and GA features.\n* Define a production readiness review gate and process for all features.\n* Utilize existing tooling with prow to enforce the process.\n\n### Non-Goals\n\n* Building new tooling to enforce the process.\n* Provide guidance for specific Kubernetes deployment models. That is,\n  requirements for features should be generally applicable to Kubernetes\n  deployments, not specific to use cases such as single or mult-tenant, cloud\n  provider, on-prem, edge, or other modes.\n\n## Proposal\n\n* Document production readiness criteria in a process document in the\n  kubernetes/community repository. Different levels of readiness may be\n  specified for different feature maturity levels.\n\n* Develop a production readiness questionnaire to ensure that the feature\n  authors consider and document operational aspects of the feature. The results\n  of this questionnaire will be included in playbook for the feature (the\n  creation of this playbook should be one of the production readiness criteria).\n\n  See [current questionnaire](https://github.com/kubernetes/community/blob/master/sig-architecture/production-readiness.md#questionnaire).\n\n* Establish a production readiness review subproject under SIG Architecture,\n  with subproject owners:\n  - johnbelamaric\n\n* Establish a production readiness review team, label, and CI check to prevent\n  the merging of feature promotion PRs that lack production readiness.\n\n### Risks and Mitigations\n\nThe primary risk is the slowing of feature merges. When this is due to the need\nfor the developers to improve the quality of the feature, that is appropriate.\nWhen this is due to lack of bandwidth in the production readiness review team,\nthat is harmful. To mitigate this, the implementation of this process must\ninclude a means of:\n * Identifying potential community members for participation in the team\n * A shadow program or other mechanism for preparing those individuals for\n   membership on the team\n * Clear criteria for when one of these individuals is ready to become a full\n   participant\n * Measurement of:\n   - Review throughput for production readiness reviews\n   - Team bench depth and size\n\n## Design Details\n\nPhase 1 - Research and Pilot\n* Targeted to the 1.17 cycle.\n* Setup a pilot PRR team, which will:\n  * Deliver an initial PRR questionnaire and pilot it with non-blocking PRRs for\n    in-progress KEPs.\n  * Deliver an interview/questionnaire form for operators and interview them on\n    production issues that they experience, to ensure that the focus of this\n    effort is on meaningful problems.\n  * Deliver a postmortem summary of existing features that have stalled due to\n    production-readiness issues (e.g., cron jobs).\n* Resolve open questions, including:\n  * ~~Should the scope of this expand to feature lifecycle?~~ No, not at this\n    time.\n  * How do we measure the effectiveness of this effort?\n\n\nPhase 2 - Implementation\n* Details TBD based on outcome of Phase 1.\n* Develop the complete PRR questionnaire.\n* Design and implement the tooling to identify features needing PRR review.\n\n## Implementation History\n\n- 2019-07-31: Created\n- 2019-10-17: Review feedback, phase 1 implementable\n- 2019-11-12: Add establishment of subproject\n"
+  },
+  {
+    "id": "e3efd998069218e29d594401ba7d9b71",
+    "title": "Ensure Conformance Tests Do Not Require Beta APIs or Features",
+    "authors": ["@liggitt"],
+    "owningSig": "sig-architecture",
+    "participatingSigs": ["sig-testing", "sig-api-machinery"],
+    "reviewers": ["@deads2k", "@bentheelder", "@timothysc", "@smarterclayton", "@johnbelamaric"],
+    "approvers": ["@timothysc", "@smarterclayton", "@johnbelamaric"],
+    "editor": "",
+    "creationDate": "2019-10-23",
+    "lastUpdated": "2019-10-23",
+    "status": "implementable",
+    "seeAlso": ["/keps/sig-architecture/20190412-conformance-behaviors.md"],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Ensure Conformance Tests Do Not Require Beta REST APIs or Features\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Story 1](#story-1)\n    - [Story 2](#story-2)\n    - [Story 3](#story-3)\n- [Design Details](#design-details)\n  - [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Infrastructure Needed](#infrastructure-needed)\n- [References](#references)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e., [kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be updated for the enhancement to be released.\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [x] Design details are appropriately documented\n- [x] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [x] Graduation criteria is in place\n- [x] \"Implementation History\" section is up-to-date for milestone\n- [x] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n- [x] KEP approvers have set the KEP status to `implementable`\n- [ ] ~~User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]~~\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nThe KEP describes a process for ensuring that the Kubernetes components and the Kubernetes conformance tests have no dependencies on beta REST APIs or features.\n\n## Motivation\n\nAs the Kubernetes project matures, and is used as the foundation for other projects,\nproduction distributions are expected to increase in stability, reliability, and consistency.\nThe Kubernetes conformance project is an important aspect of ensuring consistency across distributions.\nA production distribution should not be required to enable non-GA features or REST APIs in order to pass conformance tests.\n\nBeta dependencies can be grouped into several categories:\n* Kubernetes components requiring beta REST APIs with no stable alternatives (kubelet's use of the beta CertificateSigningRequest endpoints for cert rotation is a borderline example; cert rotation is not a required feature, but all our setup and test tools make use of it)\n* Kubernetes components requiring beta REST APIs with stable alternatives (for example, kube-scheduler accidentally switched to using the beta Events REST API in 1.16)\n* Kubernetes components requiring behavior enabled by beta feature gates\n* Kubernetes conformance tests requiring calls to beta REST APIs\n* Kubernetes conformance tests exercising behavior enabled by beta feature gates\n\nFor each category, we should identify and resolve existing dependencies, and prevent future occurrences.\n\n### Goals\n\n* Identify existing beta REST APIs and features required by Kubernetes components so they can be graduated or the dependencies removed\n* Identify existing beta REST APIs and features required by conformance tests so the tests can be rewritten to remove those dependencies\n* Prevent new beta REST APIs and features from being required by Kubernetes components or conformance tests\n* Demonstrate passing conformance tests on a cluster with all beta REST APIs and features disabled\n\n### Non-Goals\n\n* Resolve questions about conformance profiles (see https://github.com/kubernetes/community/issues/2651)\n* Resolve questions about GA-but-optional features (see https://github.com/kubernetes/community/issues/3997)\n* Resolve code-level dependencies in Kubernetes source code on pre-v1.0.0 components\n* Forbid beta REST APIs or features from being enabled in conformant clusters by distributors\n\n## Proposal\n\n### User Stories\n\n#### Story 1\n\nAs a Kubernetes component developer, I cannot accidentally make changes\nthat would require a production cluster to enable a beta API or feature\nin order to pass conformance tests.\n\nAttempts to make such a change would be caught/blocked by a CI job.\n\n#### Story 2\n\nAs a Kubernetes conformance test author, I cannot accidentally write \n(or promote to conformance) tests that require use of beta REST APIs.\n\nAttempts to make such a change would be caught/blocked by a CI job.\n\n#### Story 3\n\nAs a Kubernetes distribution maintainer, I can pass conformance tests\nwithout enabling beta REST APIs or features.\n\nA canonical CI job running against Kubernetes master and release branches\nwould demonstrate that it is possible to pass conformance tests with no\nbeta REST APIs or features enabled.\n\n## Design Details\n\n1. Make it possible to easily disable all beta REST APIs and features\n  * Add support for disabling built-in REST API versions of the form `v[0-9]+beta[0-9]+` to the Kubernetes API server with `--runtime-config api/beta=false`\n    * Parallels existing use of `--runtime-config api/all=false`\n    * For completeness, we can also add support for disabling built-in REST API versions of the form `v[0-9]+alpha[0-9]+` to the Kubernetes API server with `--runtime-config api/alpha=false`\n  * Add support for disabling beta feature gates to all components with `--feature-gates AllBeta=false`\n    * Parallels existing use of `--feature-gates AllAlpha=false`\n2. Identify existing beta REST APIs/features required by Kubernetes components or conformance tests\n  * Iteratively run a conformance test with all beta REST APIs and features disabled\n  * Identify failures due to uses of beta features/REST APIs\n  * Open issues for owners of the relevant tests or components to remove the beta dependency\n  * Construct an exemption list of beta features/REST APIs currently required to pass conformance\n3. Prevent introduction of new required beta dependencies in Kubernetes components or conformance tests\n  * Set up a merge-blocking CI job running conformance tests with all beta REST APIs and features disabled except the exemptions constructed in step 2.\n4. Resolve existing dependencies on beta REST APIs/features required by Kubernetes components or conformance tests\n  * Track issues opened in step 2\n  * As each dependency is resolved, remove it from the exemption list\n\n### Graduation Criteria\n\nPhase 1:\n* All beta REST APIs and features required to pass conformance tests are identified\n* Blocking presubmit and periodic CI jobs ensure no additional beta dependencies are introduced\n\nPhase 2:\n* All identified dependencies on beta REST APIs and features to pass conformance are resolved\n* Blocking presubmit and periodic CI jobs ensure no beta dependencies are introduced\n\nPhase 3:\n* All GA APIs required to pass conformance tests are identified\n* Blocking presubmit and periodic CI jobs ensure no dependencies on optional GA APIs are introduced as required into conformance tests\n\n## Implementation History\n\n- 2019-10-23: KEP created\n- 2019-11-01: KEP marked implementable\n\n## Infrastructure Needed\n\nA pre-submit and periodic CI job per release branch, configured with beta REST APIs and features disabled\n\n## References\n\nExisting guidelines that conformance tests should only make use of GA APIs:\n* https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/conformance-tests.md#conformance-test-requirements\n\nPast issues attempting to diagnose use of beta APIs by conformance tests:\n* https://github.com/kubernetes/kubernetes/issues/78605\n* https://github.com/kubernetes/kubernetes/issues/78613\n"
+  },
+  {
+    "id": "257bfb476fd100a85dec5dccdfb89453",
+    "title": "Bounding Self-Labeling Kubelets",
+    "authors": ["@mikedanese", "@liggitt"],
+    "owningSig": "sig-auth",
+    "participatingSigs": ["sig-node", "sig-storage"],
+    "reviewers": ["@saad-ali", "@tallclair"],
+    "approvers": ["@thockin", "@smarterclayton"],
+    "editor": "",
+    "creationDate": "2017-08-14",
+    "lastUpdated": "2018-10-31",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Bounding Self-Labeling Kubelets\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Motivation](#motivation)\n  - [Capturing Dedicated Workloads](#capturing-dedicated-workloads)\n- [Proposal](#proposal)\n- [Implementation Timeline](#implementation-timeline)\n- [Alternatives Considered](#alternatives-considered)\n  - [File or flag-based configuration of the apiserver to allow specifying allowed labels](#file-or-flag-based-configuration-of-the-apiserver-to-allow-specifying-allowed-labels)\n  - [API-based configuration of the apiserver to allow specifying allowed labels](#api-based-configuration-of-the-apiserver-to-allow-specifying-allowed-labels)\n  - [Allow kubelets to add any labels they wish, and add NoSchedule taints if disallowed labels are added](#allow-kubelets-to-add-any-labels-they-wish-and-add-noschedule-taints-if-disallowed-labels-are-added)\n  - [Forbid all labels regardless of namespace except for a specifically allowed set](#forbid-all-labels-regardless-of-namespace-except-for-a-specifically-allowed-set)\n\u003c!-- /toc --\u003e\n\n## Motivation\n\nToday the node client has total authority over its own Node labels.\nThis ability is incredibly useful for the node auto-registration flow.\nThe kubelet reports a set of well-known labels, as well as additional\nlabels specified on the command line with `--node-labels`.\n\nWhile this distributed method of registration is convenient and expedient, it\nhas two problems that a centralized approach would not have. Minorly, it makes\nmanagement difficult. Instead of configuring labels in a centralized\nplace, we must configure `N` kubelet command lines. More significantly, the\napproach greatly compromises security. Below are two straightforward escalations\non an initially compromised node that exhibit the attack vector.\n\n### Capturing Dedicated Workloads\n\nSuppose company `foo` needs to run an application that deals with PII on\ndedicated nodes to comply with government regulation. A common mechanism for\nimplementing dedicated nodes in Kubernetes today is to set a label or taint\n(e.g. `foo/dedicated=customer-info-app`) on the node and to select these\ndedicated nodes in the workload controller running `customer-info-app`.\n\nSince the nodes self reports labels upon registration, an intruder can easily\nregister a compromised node with label `foo/dedicated=customer-info-app`. The\nscheduler will then bind `customer-info-app` to the compromised node potentially\ngiving the intruder easy access to the PII.\n\nThis attack also extends to secrets. Suppose company `foo` runs their outward\nfacing nginx on dedicated nodes to reduce exposure to the company's publicly\ntrusted server certificates. They use the secret mechanism to distribute the\nserving certificate key. An intruder captures the dedicated nginx workload in\nthe same way and can now use the node certificate to read the company's serving\ncertificate key.\n\n## Proposal\n\n1. Modify the `NodeRestriction` admission plugin to prevent Kubelets from self-setting labels\nwithin the `k8s.io` and `kubernetes.io` namespaces *except for these specifically allowed labels/prefixes*:\n\n    ```\n    kubernetes.io/hostname\n    kubernetes.io/instance-type\n    kubernetes.io/os\n    kubernetes.io/arch\n\n    beta.kubernetes.io/instance-type\n    beta.kubernetes.io/os\n    beta.kubernetes.io/arch\n\n    failure-domain.beta.kubernetes.io/zone\n    failure-domain.beta.kubernetes.io/region\n\n    failure-domain.kubernetes.io/zone\n    failure-domain.kubernetes.io/region\n\n    [*.]kubelet.kubernetes.io/*\n    [*.]node.kubernetes.io/*\n    ```\n\n2. Reserve and document the `node-restriction.kubernetes.io/*` label prefix for cluster administrators\nthat want to label their `Node` objects centrally for isolation purposes.\n\n    \u003e The `node-restriction.kubernetes.io/*` label prefix is reserved for cluster administrators\n    \u003e to isolate nodes. These labels cannot be self-set by kubelets when the `NodeRestriction`\n    \u003e admission plugin is enabled.\n\nThis accomplishes the following goals:\n\n- continues allowing people to use arbitrary labels under their own namespaces any way they wish\n- supports legacy labels kubelets are already adding\n- provides a place under the `kubernetes.io` label namespace for node isolation labeling\n- provide a place under the `kubernetes.io` label namespace for kubelets to self-label with kubelet and node-specific labels\n\n## Implementation Timeline\n\nv1.13:\n\n* Kubelet deprecates setting `kubernetes.io` or `k8s.io` labels via `--node-labels`, \nother than the specifically allowed labels/prefixes described above,\nand warns when invoked with `kubernetes.io` or `k8s.io` labels outside that set.\n* NodeRestriction admission prevents kubelets from adding/removing/modifying `[*.]node-restriction.kubernetes.io/*` labels on Node *create* and *update*\n* NodeRestriction admission prevents kubelets from adding/removing/modifying `kubernetes.io` or `k8s.io`\nlabels other than the specifically allowed labels/prefixes described above on Node *update* only\n\nv1.14:\n\n* Begin migration/removal of in-tree `--node-labels` use outside of the allowed set by addons:\n  * `beta.kubernetes.io/fluentd-ds-ready`\n    * addon: remove from the nodeSelector\n    * kube-up: remove from the default `--node-labels` flag\n  * `beta.kubernetes.io/metadata-proxy-ready`\n    * addon: announce the nodeSelector will switch to `cloud.google.com/metadata-proxy-ready` in 1.15\n    * kube-up: add `cloud.google.com/metadata-proxy-ready=true` along with the existing label to `--node-labels`\n    * kube-up: add `cloud.google.com/metadata-proxy-ready=true` to existing nodes with the `beta.kubernetes.io/metadata-proxy-ready=true` label\n  * `beta.kubernetes.io/kube-proxy-ds-ready`\n    * addon: announce the nodeSelector will switch to `node.kubernetes.io/kube-proxy-ds-ready` in 1.15\n    * kube-up: add `node.kubernetes.io/kube-proxy-ds-ready=true` along with the existing label to `--node-labels`\n    * kube-up: add `node.kubernetes.io/kube-proxy-ds-ready=true` to existing nodes with the `beta.kubernetes.io/kube-proxy-ds-ready=true` label\n  * `beta.kubernetes.io/masq-agent-ds-ready`\n    * addon: announce the nodeSelector will switch to `node.kubernetes.io/masq-agent-ds-ready` in 1.16\n    * kube-up: add `node.kubernetes.io/masq-agent-ds-ready=true` to existing nodes with the `beta.kubernetes.io/masq-agent-ds-ready=true` label\n\nv1.16:\n\n* Complete migration/removal of in-tree `--node-labels` use outside of the allowed set by addons:\n  * `beta.kubernetes.io/metadata-proxy-ready`\n    * addon: change the nodeSelector to `cloud.google.com/metadata-proxy-ready`\n    * kube-up: stop setting `beta.kubernetes.io/metadata-proxy-ready`\n  * `beta.kubernetes.io/kube-proxy-ds-ready`\n    * addon: change the nodeSelector to `node.kubernetes.io/kube-proxy-ds-ready`\n    * kube-up: stop setting `beta.kubernetes.io/kube-proxy-ds-ready`\n  * `beta.kubernetes.io/masq-agent-ds-ready`\n    * addon: change the nodeSelector to `node.kubernetes.io/masq-agent-ds-ready`\n* Kubelet removes the ability to set `kubernetes.io` or `k8s.io` labels via `--node-labels`\nother than the specifically allowed labels/prefixes described above (deprecation period\nof 6 months for CLI elements of admin-facing components is complete)\n\nv1.19:\n\n* NodeRestriction admission prevents kubelets from adding/removing/modifying `kubernetes.io` or `k8s.io`\nlabels other than the specifically allowed labels/prefixes described above on Node *update* and *create*\n(oldest supported kubelet running against a v1.19 apiserver is v1.17)\n\n## Alternatives Considered\n\n### File or flag-based configuration of the apiserver to allow specifying allowed labels\n\n* A fixed set of labels and label prefixes is simpler to reason about, and makes every cluster behave consistently\n* File-based config isn't easily inspectable to be able to verify enforced labels\n* File-based config isn't easily kept in sync in HA apiserver setups\n\n### API-based configuration of the apiserver to allow specifying allowed labels\n\n* A fixed set of labels and label prefixes is simpler to reason about, and makes every cluster behave consistently\n* An API object that controls the allowed labels is a potential escalation path for a compromised node\n\n### Allow kubelets to add any labels they wish, and add NoSchedule taints if disallowed labels are added\n\n* To be robust, this approach would also likely involve a controller to automatically inspect labels and remove the NoSchedule taint. This seemed overly complex. Additionally, it was difficult to come up with a tainting scheme that preserved information about which labels were the cause.\n\n### Forbid all labels regardless of namespace except for a specifically allowed set\n\n* This was much more disruptive to existing usage of `--node-labels`.\n* This was much more difficult to integrate with other systems allowing arbitrary topology labels like CSI.\n* This placed restrictions on how labels outside the `kubernetes.io` and `k8s.io` label namespaces could be used, which didn't seem proper.\n"
+  },
+  {
+    "id": "10ca90342c2efc523613332ea64cd3de",
+    "title": "Dynamic Audit Configuration",
+    "authors": ["@pbarker"],
+    "owningSig": "sig-auth",
+    "participatingSigs": ["sig-api-machinery"],
+    "reviewers": ["@tallclair", "@yliaog", "@caesarxuchao", "@liggitt"],
+    "approvers": ["@tallclair", "@liggitt", "@yliaog"],
+    "editor": "TBD",
+    "creationDate": "2018-05-18",
+    "lastUpdated": "2018-07-31",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Dynamic Audit Control\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Dynamic Configuration](#dynamic-configuration)\n    - [Cluster Scoped Configuration](#cluster-scoped-configuration)\n  - [User Stories](#user-stories)\n    - [Story 1](#story-1)\n    - [Story 2](#story-2)\n    - [Story 3](#story-3)\n    - [Story 4](#story-4)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n    - [Feature Gating](#feature-gating)\n    - [Policy Enforcement](#policy-enforcement)\n    - [Aggregated Servers](#aggregated-servers)\n  - [Risks and Mitigations](#risks-and-mitigations)\n    - [Privilege Escalation](#privilege-escalation)\n    - [Leaked Resources](#leaked-resources)\n    - [Webhook Authentication](#webhook-authentication)\n    - [Performance](#performance)\n- [Test Plan](#test-plan)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n  - [Generalized Dynamic Configuration](#generalized-dynamic-configuration)\n  - [Policy Override](#policy-override)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nWe want to allow the advanced auditing features to be dynamically configured. Following in the same vein as \n[Dynamic Admission Control](https://kubernetes.io/docs/admin/extensible-admission-controllers/) we would like to provide \na means of configuring the auditing features post cluster provisioning.\n\n## Motivation\n\nThe advanced auditing features are a powerful tool, yet difficult to configure. The configuration requires deep insight \ninto the deployment mechanism of choice and often takes many iterations to configure properly requiring a restart of \nthe apiserver each time. Moreover, the ability to install addon tools that configure and enhance auditing is hindered \nby the overhead in configuration. Such tools frequently run on the cluster requiring future knowledge of how to reach \nthem when the cluster is live. These tools could enhance the security and conformance of the cluster and its applications.\n\n### Goals\n- Provide an api and set of objects to configure the advanced auditing kube-apiserver configuration dynamically\n\n### Non-Goals\n- Provide a generic interface to configure all kube-apiserver flags\n- configuring non-webhook backends\n- configuring audit output (format or per-field filtering)\n- authorization of audit output\n\n## Proposal\n\n### Dynamic Configuration\nA new dynamic audit backend will be introduced that follows suit with the existing [union backend](https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apiserver/pkg/audit/union.go). It will hold a map of configuration objects that it syncs with an informer.\n\n#### Cluster Scoped Configuration\nA cluster scoped configuration object will be provided that applies to all events in the cluster.\n\n```golang\n// AuditConfiguration represents a dynamic audit configuration\ntype AuditConfiguration struct {\n    metav1.TypeMeta\n\n    v1.ObjectMeta\n\n    // Policy is the current audit v1beta1 Policy object\n    // if undefined it will default to the statically configured cluster policy if available\n    // if neither exist the backend will fail\n    Policy *Policy\n\n    // Backend to send events\n    Backend *Backend\n}\n\n// Backend holds the configuration for the backend\ntype Backend struct {\n    // Webhook holds the webhook backend\n    Webhook *WebhookBackend\n}\n\n// WebhookBackend holds the configuration of the webhooks\ntype WebhookBackend struct {\n    // InitialBackoff is amount of time to wait before retrying the first failed request in seconds\n    InitialBackoff *int\n\n    // ThrottleBurst is the maximum number of events sent at the same moment\n    ThrottleBurst *int\n\n    // ThrottleEnabled determines whether throttling is enabled\n    ThrottleEnabled *bool\n\n    // ThrottleQPS maximum number of batches per second\n    ThrottleQPS *float32\n\n    // ClientConfig holds the connection parameters for the webhook\n    ClientConfig WebhookClientConfig\n}\n\n// WebhookClientConfig contains the information to make a TLS\n// connection with the webhook; this follows: \n// https://github.com/kubernetes/api/blob/master/admissionregistration/v1beta1/types.go#L222\n// but may require some additive auth parameters\ntype WebhookClientConfig struct {\n    // URL of the server\n    URL *string\n\n    // Service name to send to\n    Service *ServiceReference\n\n    // `caBundle` is a PEM encoded CA bundle which will be used to validate\n    // the webhook's server certificate.\n    CABundle []byte\n}\n```\n\nMultiple definitions can exist as independent solutions. These updates will require the audit API to be registered with the apiserver. The dynamic configurations will be wrapped by truncate and batch options, which are set statically through existing flags. Dynamic configuration will be enabled by a feature gate for pre-stable releases. If existing flags are provided to configure the audit backend they will be taken as a separate backend configuration.\n\nExample configuration yaml config:   \n```yaml\napiVersion: audit.k8s.io/v1beta1\nkind: AuditConfiguration\nmetadata:\n  name: \u003cname\u003e\npolicy:\n  rules:\n  - level: \u003clevel\u003e\n  omitStages:\n  - stage: \u003cstage\u003e\nbackend:\n  webhook:\n  - initialBackoff: \u003c10s\u003e\n    throttleBurst: \u003c15\u003e\n    throttleEnabled: \u003ctrue\u003e\n    throttleQPS: \u003c10\u003e\n    clientConfig:\n      url: \u003cbackend url\u003e\n      service: \u003coptional service name\u003e\n      caBundle: \u003cca bundle\u003e\n```\nA configuration flag will be added that enables dynamic auditing `--audit-dynamic-configuration`, which will default to false.\n\n### User Stories\n\n#### Story 1\nAs a cluster admin, I will easily be able to enable the internal auditing features of an existing cluster, and tweak the configurations as necessary. I want to prevent privilege escalation from being able to tamper with a root audit configuration.\n\n#### Story 2\nAs a Kubernetes extension developer, I will be able to provide drop in extensions that utilize audit data.\n\n#### Story 3\nAs a cluster admin, I will be able configure multiple audit-policies and webhook endpoints to provide independent auditing facilities.\n\n#### Story 4\nAs a kubernetes developer, I will be able to quickly turn up the audit level on a certain area to debug my application.\n\n### Implementation Details/Notes/Constraints\n\n#### Feature Gating\nIntroduction of dynamic policy requires changes to the current audit pipeline. Care must be taken that these changes are \nproperly gated and do not affect the stability or performance of the current features as they progress to GA. A new decorated \nhandler will be provisioned similar to the [existing handlers](https://github.com/kubernetes/apiserver/blob/master/pkg/endpoints/filters/audit.go#L41) \ncalled `withDynamicAudit`. Another conditional clause will be added where the handlers are \n[provisioned](https://github.com/kubernetes/apiserver/blob/master/pkg/server/config.go#L536) allowing for the proper feature gating.\n\n#### Policy Enforcement\nThis addition will move policy enforcement from the main handler to the backends. From the `withDynamicAudit` handler, \nthe full event will be generated and then passed to the backends. Each backend will copy the event and then be required to \ndrop any pieces that do not conform to its policy. A new sink interface will be required for these changes called `EnforcedSink`, \nthis will largely follow suite with the existing sink but take a fully formed event and the authorizer attributes as its \nparameters. It will then utilize the `LevelAndStages` method in the policy \n[checker](https://github.com/kubernetes/apiserver/blob/master/pkg/audit/policy/checker.go) to enforce its policy on the event, \nand drop any unneeded sections. The new dynamic backend will implement the `EnforcedSink` interface, and update its state \nbased on a shared informer. For the existing backends to comply, a method will be added that implements the `EnforcedSink` interface.\n\nImplementing the [attribute interface](https://github.com/kubernetes/kubernetes/tree/master/staging/src/k8s.io/apiserver/pkg/authorization/authorizer/interfaces.go) \nbased on the Event struct was also explored. This would allow us to keep the existing `Sink` interfaces, however it would \nrequire parsing the request URI twice in the pipeline due to how that field is represented in the Event. This was determined \nto not be worth the cost.\n\n#### Aggregated Servers\nInherently apiserver aggregates and HA apiserver setups will work off the same dynamic configuration object. If separate \naudit configuration objects are needed they should be configured as static objects on the node and set through the runtime flags. Aggregated servers will implement the same audit handling mechanisms. A conformance test should be provided as assurance. Metadata level \nlogging will happen by default at the main api server as it proxies the traffic. The aggregated server will then watch the same \nconfiguration objects and only log on resource types that it handles. This will duplicate the events sent to the receiving servers \nso they should not expect to key off `{ Audit-ID x Stage }`.\n\n### Risks and Mitigations\n\n#### Privilege Escalation\nThis does open up the attack surface of the audit mechanisms. Having them strictly configured through the api server has the advantage of limiting the access of those configurations to those that have access to the master node. This opens a number of potential attack vectors:   \n\n* privileged user changes audit policy to hide (not audit) malicious actions\n* privileged user changes audit policy to DoS audit endpoint (with malintent, or ignorance)\n* privileged user changes webhook configuration to hide malicious actions\n\nAs a mitigation strategy policy configured through a static file on the api server will not be accessible through the api. This file ensures that an escalation attack cannot tamper with a root configuration, but works independently of any dynamically configured objects.\n\n#### Leaked Resources\nA user with permissions to create audit policies effectively has read access to the entire cluster (including all secrets data).\n\nA mitigation strategy will be to document the exposure space granted with this resource. Advice will be provided to only allow access to cluster admin level roles.\n\n#### Webhook Authentication\nWith Dynamic Admission control today any authentication mechanism must be provided through a static kubeconfig file on the node. This hinders a lot of the advances in this proposal. All webhooks would require authentication as an unauthenticated endpoint would allow a bad actor to push phony events. Lack of dynamic credential provisioning is problematic to the drop-in extension use case, and difficult to configure.\n\nThe reason for static configuration today is that a single configured credential would have no way of differentiating apiserver replicas or their aggregates. There is a possible mitigation by providing a bound service account token and using the calling server's dns name as the audience.\n\nIt may also be reasonable to provide a dynamic auth configuration from secrets, with the understanding that it is shared by the api servers.\n\nThis needs further discussion.\n\n#### Performance\n\nThese changes will likely have an O(n) performance impact on the api server per policy.  A `DeepCopy` of the event will be \nrequired for each backend. Also, the request/response object would now be serialized on every [request](https://github.com/kubernetes/kubernetes/blob/cef2d325ee1be894e883d63013f75cfac5cb1246/staging/src/k8s.io/apiserver/pkg/audit/request.go#L150-L152). \nBenchmark testing will be required to understand the scope of the impact and what optimizations may be required. This impact \nis gated by opt-in feature flags, which allows it to move to alpha but these concerns must be tested and reconciled before it \nprogresses to beta.\n\n## Test Plan\n\nThere are tests for the [plugin](https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/dynamic_test.go) and any other changes to the codebase. There is an [integration test](https://github.com/kubernetes/kubernetes/blob/master/test/integration/master/audit_dynamic_test.go) as well as an [e2e test](https://github.com/kubernetes/kubernetes/blob/master/test/e2e/auth/audit_dynamic.go).\n\nBefore reaching GA, a scalability test should be in place.\n\n## Graduation Criteria\n\nSuccess will be determined by stability of the provided mechanisms and ease of understanding for the end user.\n\n* alpha: Api server flags can be dynamically configured, known issues are tested and resolved.\n* beta: Mechanisms have been hardened against any known bugs and the process is validated by the community\n\n## Implementation History\n\n- 05/18/2018: initial design\n- 06/13/2018: updated design\n- 07/31/2018: dynamic policy addition\n\n## Alternatives\n\n### Generalized Dynamic Configuration\n\nWe could strive for all kube-apiserver flags to be able to be dynamically provisioned in a common way. This is likely a large \ntask and out of the scope of the intentions of this feature.\n\n### Policy Override\n\nThere has been discussion over whether the policy configured by api server flags should limit the policies configured dynamically. \nThis would allow a cluster admin to narrowly define what is allowed to be logged by the dynamic configurations. While this has upsides \nit was ruled out for the following reasons: \n\n* It would limit user story #4 in the ability to quickly turn up logging when needed \n* It could prove difficult to understand as the policies themselves are fairly complex \n* The use of CRDs would be difficult to bound\n\nThe dynamic policy feature is gated by runtime flags. This still provides the cluster provisioner a means to limit audit logging to the \nsingle runtime object if needed.\n"
+  },
+  {
+    "id": "02f19b72b566fef00b6bb7ece9d0ddfc",
+    "title": "Harden Default RBAC Discovery ClusterRole(Binding)s",
+    "authors": ["@dekkagaijin"],
+    "owningSig": "sig-auth",
+    "participatingSigs": ["sig-auth", "sig-api-machinery"],
+    "reviewers": ["@liggitt", "@tallclair", "@deads2k"],
+    "approvers": ["@liggitt", "@tallclair", "@deads2k"],
+    "editor": "TBD",
+    "creationDate": "2019-01-28",
+    "lastUpdated": "2019-01-31",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Harden Default RBAC Discovery ClusterRole(Binding)s\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Risks and Mitigations](#risks-and-mitigations)\n    - [Existing customization of \u003ccode\u003esystem:discovery\u003c/code\u003e](#existing-customization-of-)\n    - [Dependence on existing unauthenticated behavior](#dependence-on-existing-unauthenticated-behavior)\n- [Graduation Criteria](#graduation-criteria)\n  - [Testing](#testing)\n  - [Documentation](#documentation)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThe aim of this change is to remove the `system:unauthenticated` subjects from the `system:discovery` and `system:basic-user` ClusterRoleBindings, while preserving unauthenticated access to a genuinely non-sensitive subset of the current APIs (e.g. `GET /healthz`, `GET /version`). This will improve the default privacy and security posture for new clusters.\n\n## Motivation\n\nOne of the work items resulting from the [CVE-2018-1002105](https://github.com/kubernetes/kubernetes/issues/71411) post-mortem was to [investigate hardening the default RBAC discovery ClusterRoleBindings](https://github.com/kubernetes/kubernetes/issues/72115) (i.e. `system:discovery` and `system:basic-user`) to limit potential avenues for similar attack. Additionally, the fact that API extensions are exposed by the default discovery bindings is surprising to some and represents a potential privacy concern (e.g. `GET /apis/self-flying-cars.unicorn.vc/v1/`).\n\n### Goals\n\n* Remove discovery from the set of APIs which allow for unauthenticated access by default, improving privacy for CRDs and the default security posture of default clusters in general.\n\n### Non-Goals\n\n* To protect default clusters from unauthenticated access entirely (already achievable via `--anonymous-auth=false`), or to add support for more granular schema discovery.\n  * There are several non-sensitive and legitimate use-cases for unauthenticated calls, such as `/healthz` liveliness checks and [returning useful information about the cluster](https://github.com/kubernetes/kubernetes/issues/45366#issuecomment-299275002) to `kubectl version`, that we don't wish to break.\n* To prevent _namespace_ admins from granting namespace-level permissions to anonymous users.\n* To protect the CRD names and details from _authenticated_ users.\n* To address other, more fundamental, discovery-related bugs uncovered in [CVE-2018-1002105](https://github.com/kubernetes/kubernetes/issues/71411), e.g.:\n  * https://github.com/kubernetes/kubernetes/issues/72113\n  * https://github.com/kubernetes/kubernetes/issues/72117\n\n## Proposal\n\n* Remove the `system:unauthenticated` subject group from the default [`system:discovery` and `system:basic-user` ClusterRoleBindings](https://github.com/kubernetes/kubernetes/blob/release-1.13/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go#L531-L532)\n* Create a new default ClusterRole which prominently, explicitly documents its intent to freely disclose information (e.g. `system:public-info-viewer`) and contains the `GET /healthz`, `/version` and `/version/` PolicyRules.\n* Create a new default ClusterRoleBinding for the new ClusterRole, which grants the `system:(un)authenticated` subject groups access.\n\n### Risks and Mitigations\n\n#### Existing customization of `system:discovery`\n\nIf a user upgrades a cluster which has modified the `system:discovery` ClusterRoleBinding, these changes could either be trampled or restricted endpoints could end up being re-exposed via the new `system:public-info-viewer` binding.\n\nMitigation: During the API server's RBAC auto-reconciliation, if `system:discovery` exists and `system:public-info-viewer` does not (i.e. the state after an upgrade) we'll copy `system:discovery`'s subjects and reconciliation annotation value. This will preserve permissions as-is during API server upgrades, even with the addition of `system:public-info-viewer`, since existing customization will be preserved.\n\n#### Dependence on existing unauthenticated behavior\n\nSome use-cases might require the existing permissions to be preserved for unauthenticated calls, and some currently-working configurations might be broken for new installs. \n\nDisambiguating between accidental and necessary dependence on the current behavior will have to be determined by the user on a case-by-case basis. However, in the release notes, we can include easy 'escape hatches' to re-enable unauthenticated access to the discovery APIs, such as the ones recommended to cluster admins below.\n\nFrom [`system:basic-user`](https://github.com/kubernetes/kubernetes/blob/8b98e802eddb9f478ff7d991a2f72f60c165388a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go#L209-L215):\n* create `selfsubjectaccessreviews`\n* create `selfsubjectrulesreviews`\n\npro: unauthenticated users can't list actions available to them\ncon: this doesn't actually forbid actions, and can create a potentially confusing mismatch for users which rely on these APIs as a sanity check\n\nFrom [`system:discovery`](https://github.com/kubernetes/kubernetes/blob/8b98e802eddb9f478ff7d991a2f72f60c165388a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go#L198-L208):\n* get `/api`, `/api/*`\n* get `/apis`, `/apis/*`\n* get `/openapi`, `/openapi/*`\n\npro: can't see potentially sensitive custom resource names/schemas by default\ncon: consumers that need schema info, such as editors and users of namespaced resources w/ default tools (e.g. `kubectl get pods -n foo`) must allow authentication to the discovery APIs to fetch it (though, this is already the case with clusters which have disabled anonymous auth). More concretely, **cluster admins that wish anonymous users to have API access should grant these permissions as part of cluster setup**, for example:\n```\nkubectl create clusterrolebinding anonymous-discovery --clusterrole=system:discovery --group=system:unauthenticated\nkubectl create clusterrolebinding anonymous-access-review --clusterrole=system:basic-user --group=system:unauthenticated\n```\nA potential future feature could automatically grant discovery permissions to anonymous users in the event that they're granted access to another API.\n\n## Graduation Criteria\n\nThis proposal will have 'graduated' once the unauthenticated API surface has been minimized without excessive user impact. Excessive user impact includes issues that can't be mitigated with a single `kubectl` invocation or fixed by enabling request authentication.\n\n### Testing \n\nTo address the addition of `system:public-info-viewer` and the modification of `system:discovery` and `system:basic-user`:\n* update testdata for [ClusterRoles](https://github.com/kubernetes/kubernetes/blob/master/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml) and [ClusterRoleBindings](https://github.com/kubernetes/kubernetes/blob/master/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-role-bindings.yaml)\n* add/modify cases in [reconcile_role_test.go](https://github.com/kubernetes/kubernetes/blob/master/pkg/registry/rbac/reconciliation/reconcile_role_test.go) and [reconcile_rolebindings_test.go](https://github.com/kubernetes/kubernetes/blob/master/pkg/registry/rbac/reconciliation/reconcile_rolebindings_test.go)\n\n### Documentation\n\nDocumentation regarding the [default RBAC discovery ClusterRole(Bindings)](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#discovery-roles) will have to be updated to reflect the new realities.\n\n## Implementation History\n\n2019-01-31: KEP submitted\n2019-02-06: [implementation PR opened](https://github.com/kubernetes/kubernetes/pull/73807)\n2019-02-27: [documentation PR opened](https://github.com/kubernetes/website/pull/12888)\n2019-03-01: implementation PR merged\n2019-03-11: documentation PR closed\n"
+  },
+  {
+    "id": "960c0f70b90507ff787e25672b9c951f",
+    "title": "Support external signing of service account keys",
+    "authors": ["@micahhausler"],
+    "owningSig": "sig-auth",
+    "participatingSigs": [],
+    "reviewers": ["@mikedanese", "@liggit", "@tallclair"],
+    "approvers": ["@mikedanese", "@liggit", "@tallclair"],
+    "editor": "@micahhausler",
+    "creationDate": "2019-01-16",
+    "lastUpdated": "2019-05-17",
+    "status": "implementable",
+    "seeAlso": [],
+    "replaces": [],
+    "supersededBy": [],
+    "markdown": "\n# Support external signing of service account keys\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Preserve existing behavior](#preserve-existing-behavior)\n  - [Updates to API server token generation](#updates-to-api-server-token-generation)\n  - [New API](#new-api)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThe Kubernetes API server has always read service account keys from disk as the process starts, and kept them in memory for the duration of the server's lifetime. As the API server can now verify and issue projected volume tokens, it would be advantageous to support external signing and verifying of token data over an API, as well as reading public keys from an API.\n\n## Motivation\n\nFor operators who want to regularly rotate the signing and verifying keys for projected volume tokens, the Kubernetes API server must be restarted in order to use a new key. To facilitate easy key rotation, this KEP includes an proposal for a grpc API to support out of process signing and listing of signing keys.\n\n### Goals\n\n- Support for out-of-process JWT signing\n- Support for listing public verifying keys\n- Preserve existing behavior and performance for keys not read over a socket\n\n### Non-Goals\n\n- Reading TLS serving certificates and key from a socket or reloading of the API server with new cert and key\n- Reading any other certificates from a file\n\n## Proposal\n\n### Preserve existing behavior\n\nThe API server flags `--service-account-key-file` and `--service-account-signing-key-file` will continue be used for reading from files.\n\n### Updates to API server token generation\n\nAs of Kubernetes v1.13.2, the API server uses the functions `JWTTokenGenerator` and `JWTTokenAuthenticator`. New types that implement the `TokenGenerator` interface and support token validation will be added to `k8s.io/kubernetes/pkg/serviceaccount/`.\n\n### New API\n\nI'm proposing creating a new versioned grpc API under `k8s.io/kubernetes/pkg/serviceaccount`. This will be similar to how the KMS envelope encryption has an API at `k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1/service.proto`\n\n```proto\nsyntax = \"proto3\";\n\npackage v1alpha1;\n\nservice KeyService {\n  // Sign an incoming payload\n  rpc SignPayload(SignPayloadRequest) returns (SignPayloadResponse) {}\n  // List all active public keys\n  rpc ListPublicKeys(ListPublicKeysRequest) returns (ListPublicKeysResponse) {}\n}\n\nmessage SignPayloadRequest {\n  // payload is the content to be signed. JWT headers must be included by the caller\n  bytes payload = 1;\n  // algorithm specifies which algorithm to sign with\n  string algorithm = 2;\n}\nmessage SignPayloadResponse {\n  // content returns the signed payload\n  bytes content = 1;\n}\n\n\nmessage PublicKey {\n  // public_key is a PEM encoded public key\n  bytes public_key = 1;\n  // certificate is a concatenated list of PEM encoded x509 certificates\n  bytes certificates = 2;\n  // key_id is the key's ID\n  string key_id = 3;\n  // algorithm states the algorithm the key uses\n  string algorithm = 4;\n}\n\nmessage ListPublicKeysRequest {}\nmessage ListPublicKeysResponse {\n  // key_id is the key's ID\n  string active_key_id = 1;\n  // public_keys is a list of public verifying keys\n  repeated PublicKey public_keys = 2;\n}\n```\n\n### Implementation Details/Notes/Constraints\n\nThe API server flag `--service-account-key-file` can be specified multiple times for legacy SA tokens and projected tokens. Validation keys from this flag will be merged with the response of `ListPublicKeys()`. A new flag `--key-service-url` will be added to the API server specifying a unix socket where the key service will be accessible.\n\n### Risks and Mitigations\n\nNew token generation and validation could suffer a performance difference when reading over a socket, as an external process will be signing data.\n\nSigning and verifying tokens over a grpc API carries the risk of a server side request forgery, where a malicious client could generate tokens. To mitigate this risk, the API will only be accessible over a unix socket.\n\n## Graduation Criteria\n\n\u003c!-- TODO --\u003e\n\n## Implementation History\n\n* Initial PR: kubernetes/kubernetes#73110\n"
+  },
+  {
+    "id": "38a3ed13a71a08d82c0966c9b664577a",
+    "title": "Certificates API",
+    "authors": ["@mikedanese", "@deads2k"],
+    "owningSig": "sig-auth",
+    "participatingSigs": null,
+    "reviewers": ["@liggitt", "@smarterclayton", "@munnerz"],
+    "approvers": ["@liggitt", "@smarterclayton"],
+    "editor": "",
+    "creationDate": "2019-06-07",
+    "lastUpdated": "2020-01-21",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Certificates API\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n- [Design Details](#design-details)\n  - [Sequence of an Issuance](#sequence-of-an-issuance)\n  - [Signers](#signers)\n    - [Limiting approval and signer powers for certain signers.](#limiting-approval-and-signer-powers-for-certain-signers)\n  - [CertificateSigningRequest API Definition](#certificatesigningrequest-api-definition)\n  - [Manual CSR Approval With Kubectl](#manual-csr-approval-with-kubectl)\n  - [Automatic CSR Approval Implementations](#automatic-csr-approval-implementations)\n  - [Automatic Signer Implementations](#automatic-signer-implementations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThe Certificates API enables automation of\n[x509](https://tools.ietf.org/html/rfc5280) credential provisioning by providing\na programmatic interface for clients of the Kubernetes API to request and obtain\nx509 certificates from a Certificate Authority (CA).\n\n## Motivation\n\nThe security of the Kubernetes platform is underpinned by a public key\ninfrastructure (PKI). Each Kubernetes cluster has a root certificate authority.\nThis CA is used to secure communication between cluster components. The\nCertificates API was originally merged to support [Kubelet TLS\nBootstrap](https://github.com/kubernetes/community/blob/1fd524165bcf54d4bef99adb8332df72f4f88d5c/contributors/design-proposals/cluster-lifecycle/kubelet-tls-bootstrap.md)\nbut users have also begun to use this API to provision certificates for PKI\nneeds out of core.\n\n### Goals\n\n- Provide a generic API for PKI issuance to be consumed by both core Kubernetes\n  components as well as user workloads running in the cluster.\n- Support extensions that allow for specialized certificate issuance.\n\n### Non-Goals\n\n- Build in support for specialized certificate issuance (e.g.\n  [LetsEncrypt](https://letsencrypt.org/)).\n\n## Proposal\n\nWe intend to provision initial and renewed certificates in Kubernetes (often\nwithout operator intervention). The crux of this process is how to make (and\nautomate) the decision of whether to approve or deny a particular certificate\nsigning request.\n\nThe role of a Registration Authority (referred to in this design as the\napprover) is to verify that a CSR satisfies two requirements:\n\n1. Authentication: The subject of the CSR is the origin of the CSR.\n1. Authorization: The subject of the CSR is authorized to act in the requested\n   context (e.g. has authority over requested Subject Alternative Names, etc).\n\nIff these two requirements are met, the approver should approve the CSR and\notherwise should deny the CSR. Once the CSR is approved a Certificate Authority\n(referred to in this design as the signer) should construct a certificate from\nthe CSR and return the certificate to the requester.\n\nThe Certificates API provides a generic means of communication exposed via the\nKubernetes resource model over which a certificate requestor, approver and\nsigner can interact.\n\n## Design Details\n\nA client requesting a certificate post a CertificateSigningRequest to the\nCertificates API. The client may only provide the encoded [Certificate\nRequest](https://tools.ietf.org/html/rfc2986), usages of the certificate in\nthe spec, the standard object metadata, and the requested signer on the initial creation of the\nCertificateSigningRequest. The kube-apiserver also asserts authentication\nattributes of the requestor in the CertificateSigningRequest spec before\ncommitting it to storage so that they can be used later during CSR approval. The\ninformation contained in the spec is immutable after the request is created.\n\nAn approver updates approval status of the CertificateSigningRequest via the\nCertificateSigningRequestStatus. The approval condition can only be updated via\nthe `/approval` subresource allowing approval permission to be authorized\nindependently of other operations on the CertificateSigningRequest.\n\nContingent on approval, a signer posts a signed certificate to the status. The\ncertificate field of the status can only be updated via the `/status`\nsubresource allowing signing permission to be authorized independently of other\noperations on the CertificateSigningRequest.\n\nThe API is designed to support the standard asynchronous controller model of\nKubernetes where the approver and signer act as independent controllers of the\nCertificates API. Since issuance is asynchronous, an approver can perform\nout-of-band verification of the CSR before making an authorization decision.\n\nThe approver is designed to be explicitly independent of the signer. This\nseparates concerns of authorization and certificate minting and also allows the\nsigner to operate in a separate, more restrictive environment than the approver.\nThis is typical of many PKI architectures.\n\n### Sequence of an Issuance\n\nA typical successful issuance proceeds as follows.\n\n![CSR](/keps/sig-auth/csr.png)\n\n1. The requestor generates a private key, builds a certificate signing request,\n   and submits the `CertificateSigningRequest` to the Kubernetes certificates\n   API.\n1. The approver controller observes the newly submitted request, validates and\n   authorizes the request and if all goes well, approves the request.\n1. The signer observes the approval, mints a new certificate and stores it in\n   the `.Status.Certificate` field.\n1. The requestor observes the update, and stores the certificate locally.\n\n### Signers\n\nCSRs have a `signerName` field which is used to specify which signer the CSR creator wants to sign the certificate.\nTo support migration from v1beta1 to v1, this required field will be defaulted in v1beta1 (optional in openapi), but\nnot defaulted and required in v1 :\n 1. If it's a kubelet client certificate, it is assigned \"kubernetes.io/kube-apiserver-client-kubelet\".\n 2. If it's a kubelet serving certificate, it is assigned \"kubernetes.io/kubelet-serving\". \n see https://github.com/kubernetes/kubernetes/blob/release-1.10/pkg/controller/certificates/approver/sarapprove.go#L211-L223 for details.\n 3. Otherwise, it is assigned \"kubernetes.io/legacy-unknown\".\n\nThere will be field selector support to make approvers and signers easier to write.\n\nAll signers should provide information about how they work so that clients can predict what will happen to their CSRs.\nThis includes:\n 1. Trust distribution - how trust (ca bundles) are distributed.\n 2. Permitted subjects - (any? specific subtree?) and behavior when a disallowed subject is requested.\n 3. Permitted x509 extensions - (IP SANs? DNS SANs? Email SANs? URI SANs? others?) and behavior when a disallowed\n extension is requested.\n 4. Permitted key usages / extended key usages - (client only? server only? any? signer-determined? CSR-determined?) and\n behavior when usages different than the signer-determined usages are specified in the CSR.\n 5. Expiration/cert lifetime - (fixed by signer? configurable by admin? CSR-determined?) and behavior when an expiration\n different than the signer-determined expiration is specified in the CSR.\n 6. CA bit allowed/disallowed - and behavior if a CSR contains a request a for a CA cert when the signer does not permit it.\n 7. (optional) Information about the meaning of additional CERTIFICATE PEM blocks in `status.certificate`, if different from the\n standard behavior of treating the additional certificates as intermediates, and presenting them in TLS handshakes.\n\n\nsig-auth reserves all `kubernetes.io/*` `signerNames` and more may be added in the future.\nKubernetes provides the following well-known signers.  Today, failures for all of these are only reported in kube-controller-manager logs:\n 1. kubernetes.io/kube-apiserver-client - signs certificates that will be honored as client-certs by the kube-apiserver.\n    Never auto-approved by kube-controller-manager.\n    1. Trust distribution: signed certificates must be honored as client-certificates by the kube-apiserver.  The CA bundle\n       is not distributed by any other means.\n    2. Permitted subjects - no subject restrictions, but approvers and signers may choose not to approve or sign.\n       Certain subjects like cluster-admin level users or groups vary between distributions and installations, but deserve\n       additional scrutiny before approval and signing.  An admission plugin is available to restrict system:masters, but\n       it is often not the only cluster-admin subject in a cluster.\n    3. Permitted x509 extensions - Non-critical extensions should be dropped.\n    4. Permitted key usages - must include `[]string{\"client auth\"}`.  Must not include key usages beyond `[]string{\"digital signature\", \"key encipherment\", \"client auth\"}`\n    5. Expiration/cert lifetime - minimum of CSR signer or request.  Sanity of the time is the concern of the signer.\n    6. CA bit allowed/disallowed - not allowed.\n 2. kubernetes.io/kube-apiserver-client-kubelet - signs client certificates that will be honored as client-certs by the kube-apiserver.\n    May be auto-approved by kube-controller-manager.\n    1. Trust distribution: signed certificates must be honored as client-certificates by the kube-apiserver.  The CA bundle\n       is not distributed by any other means.\n    2. Permitted subjects - organizations are exactly `[]string{\"system:nodes\"}`, common name starts with `\"system:node:\"`\n    3. Permitted x509 extensions - none\n    4. Permitted key usages - exactly `[]string{\"key encipherment\", \"digital signature\", \"client auth\"}`\n    5. Expiration/cert lifetime - minimum of CSR signer or request.  Sanity of the time is the concern of the signer.\n    6. CA bit allowed/disallowed - not allowed.\n 3. kubernetes.io/kubelet-serving - signs serving certificates that are honored as a valid kubelet serving certificate \n    by the kube-apiserver, but has no other guarantees.  Never auto-approved by kube-controller-manager.\n    1. Trust distribution: signed certificates must be honored by the kube-apiserver as valid to terminate connections to a kubelet.\n       The CA bundle is not distributed by any other means.\n    2. Permitted subjects - organizations are exactly `[]string{\"system:nodes\"}`, common name starts with `\"system:node:\"`\n    3. Permitted x509 extensions - DNS and IP SANs are allowed\n    4. Permitted key usages - exactly `[]string{\"key encipherment\", \"digital signature\", \"server auth\"}`\n    5. Expiration/cert lifetime - minimum of CSR signer or request.\n    6. CA bit allowed/disallowed - not allowed.\n 4. kubernetes.io/legacy-unknown - has no guarantees for trust at all.  Some distributions may honor these as client\n    certs, but that behavior is not standard kubernetes behavior.  Never auto-approved by kube-controller-manager.\n    1. Trust distribution: None.  There is no standard trust or distribution for this signer in a kubernetes cluster.\n    2. Permitted subjects - any\n    3. Permitted x509 extensions - honors SAN extensions and discards other extensions.\n    4. Permitted key usages - any\n    5. Expiration/cert lifetime - minimum of CSR signer or request.  Sanity of the time is the concern of the signer.\n    6. CA bit allowed/disallowed - not allowed.\n\nDistribution of trust happens out of band for these signers.  Any trust outside of those described above are strictly\ncoincidental.  For instance, some distributions may honor kubernetes.io/legacy-unknown as client-certificates for the\nkube-apiserver, but this is not a standard.\nNone of these usages are related to ServiceAccount token secrets `.data[ca.crt]` in any way.  That ca-bundle is only\nguaranteed to verify a connection the kube-apiserver using the default service.\n\nTo support HA upgrades, the kube-controller-manager will duplicate defaulting code for an empty `signerName` for one\nrelease.\n\n#### Limiting approval and signer powers for certain signers.\nGiven multiple signers which may be implemented as \"dumb\" controllers that sign if the CSR is approved, there is benefit\nto providing a simple way to subdivide approval powers through the API.  We will introduce an admission plugin that requires\n 1. verb == `approve`\n 2. resource == `signers`\n 3. name == `\u003c.spec.signerName\u003e` \n 4. group == `certificates.k8s.io`\n \nTo support a use-case that wants a single rule to allow approving an entire domain (example.com in example.com/cool-signer),\nthere will be a second check for\n 1. verb == `approve`\n 2. resource == `signers`\n 3. name == `\u003c.spec.signerName domain part only\u003e/*` \n 4. group == `certificates.k8s.io`\n\nThere are congruent check for providing a signature that use the verb==\"sign\" instead of \"approve\" above.\n\nFor migration, we will provide three bootstrap cluster-roles defining authorization rules needed to approve CSRs for the kubernetes.io signerNames.\nCluster admins can either:\n1. grant signer-specific approval permissions using roles they define\n2. grant signer-specific approval permissions using the bootstrap roles starting in 1.18\n3. disable the approval-authorizing admission plugin in 1.18 (if they don't care about partitioning approver rights)\n \n\n### CertificateSigningRequest API Definition\n\n```go\ntype CertificateSigningRequest struct {\n  // spec information is immutable after the request is created.\n  // Only the request, usages, and signerName fields can be set on creation,\n  // other fields are derived by Kubernetes and cannot be modified by users.\n  Spec   CertificateSigningRequestSpec\n  Status CertificateSigningRequestStatus\n}\n\ntype CertificateSigningRequestSpec struct {\n  // requested signer for the request up to 571 characters long.  It is a qualified name in the form: `scope-hostname.io/name`.  \n  // If empty, it will be defaulted for v1beta1:\n  //  1. If it's a kubelet client certificate, it is assigned \"kubernetes.io/kube-apiserver-client-kubelet\".  This is determined by \n  //     Seeing if organizations are exactly `[]string{\"system:nodes\"}`, common name starts with `\"system:node:\"`, and\n  //     key usages are exactly `[]string{\"key encipherment\", \"digital signature\", \"client auth\"}`\n  //  2. Otherwise, it is assigned \"kubernetes.io/legacy-unknown\".\n  // In v1 it will be required.  Distribution of trust for signers happens out of band. \n  // The following signers are known to the kube-controller-manager signer.\n  //  1. kubernetes.io/kube-apiserver-client - signs certificates that will be honored as client-certs by the kube-apiserver. Never auto-approved by kube-controller-manager.\n  //  2. kubernetes.io/kube-apiserver-client-kubelet - signs client certificates that will be honored as client-certs by the kube-apiserver. May be auto-approved by kube-controller-manager.\n  //  3. kubernetes.io/kubelet-serving - signs serving certificates that are honored as a valid kubelet serving certificate by the kube-apiserver, but has no other guarantees.\n  //  4. kubernetes.io/legacy-unknown - has no guarantees for trust at all.  Some distributions may honor these as client certs, but that behavior is not standard kubernetes behavior.\n  // None of these usages are related to ServiceAccount token secrets `.data[ca.crt]` in any way.\n  // You can select on this field using `.spec.signerName`.\n  SignerName string\n\n  // Base64-encoded PKCS#10 CSR data\n  Request []byte\n\n  // usages specifies a set of usage contexts the key will be\n  // valid for.\n  // See: https://tools.ietf.org/html/rfc5280#section-4.2.1.3\n  //      https://tools.ietf.org/html/rfc5280#section-4.2.1.12\n  Usages []KeyUsage\n\n  // Information about the requesting user.\n  // See user.Info interface for details.\n  Username string\n  // UID information about the requesting user.\n  // See user.Info interface for details.\n  UID string\n  // Group information about the requesting user.\n  // See user.Info interface for details.\n  Groups []string\n  // Extra information about the requesting user.\n  // See user.Info interface for details.\n  Extra map[string]ExtraValue\n}\n\n// ExtraValue masks the value so protobuf can generate\ntype ExtraValue []string\n\ntype CertificateSigningRequestStatus struct {\n  // Conditions applied to the request, such as approval or denial.\n  Conditions []CertificateSigningRequestCondition\n\n  // Certificate is populated by the signer with the issued certificate in PEM format.\n  // In JSON and YAML output, this entire field is base64-encoded, so it consists of:\n  // base64(\n  // MII...Pb7Yu/E=\n  // optional intermediate certificate blocks\n  // MII...MGKB\n  // MII...AY1M\n  // )\n  //\n  // In v1beta1, this field is unvalidated.\n  //\n  // In v1, modified content in this field is validated:\n  // * Content must contain one or more PEM blocks\n  // * All PEM blocks must have the \"CERTIFICATE\" label, contain no headers, and the encoded data\n  //   must be a BER-encoded ASN.1 Certificate structure as described in section 4 of RFC5280.\n  // * Non-PEM content may appear before or after the CERTIFICATE PEM blocks and is unvalidated,\n  //   to allow for explanatory text as described in section 5.2 of RFC7468.\n  //\n  // If more than one PEM block is present, and the definition of the requested spec.signerName\n  // does not indicate otherwise, the first block is the issued certificate,\n  // and subsequent blocks should be treated as intermediate certificates and presented in TLS handshakes.\n  Certificate []byte\n}\n\ntype CertificateSigningRequestCondition struct {\n  // request approval state, currently Approved or Denied.\n  Type RequestConditionType\n  // brief reason for the request state\n  Reason string\n  // human readable message with details about the request state\n  Message string\n}\n\ntype RequestConditionType string\n\n// These are the possible conditions for a certificate request.\nconst (\n  CertificateApproved RequestConditionType = \"Approved\"\n  CertificateDenied   RequestConditionType = \"Denied\"\n)\n```\n\n### Manual CSR Approval With Kubectl\n\nA Kubernetes administrator (with appropriate permissions) can manually approve\n(or deny) Certificate Signing Requests by using the `kubectl certificate\napprove` and `kubectl certificate deny` commands.\n\n### Automatic CSR Approval Implementations\n\nThe kube-controller-manager ships with an in-built\n[approver](https://github.com/kubernetes/kubernetes/blob/32ec6c212ec9415f604ffc1f4c1f29b782968ff1/pkg/controller/certificates/approver/sarapprove.go)\nfor Kubelet TLS Bootstrap that delegates various permissions on CSRs for node\ncredentials to authorization. It does this by posting subject access reviews to\nthe API server. It punts on TLS certificates for server authentication of the\nKubelet API because verifying IP SANs for Kubelets in a generic way poses\nchallenges.\n\nThe GCP controller manager replaces the in-built approver with an\n[approver](https://github.com/kubernetes/cloud-provider-gcp/blob/08fa1e3260ffb267682762e24ba93692000e3be8/cmd/gcp-controller-manager/csr_approver.go)\nthat handles TLS certificates for server authentication and support for\nverification of CSRs with proof of control of GCE vTPM.\n\nAn external project, [kapprover](https://github.com/coreos/kapprover), does\npolicy based approval of kubelet CSRs.\n\n### Automatic Signer Implementations\n\nThe kube-controller-manager ships with an in-built\n[signer](https://github.com/kubernetes/kubernetes/blob/32ec6c212ec9415f604ffc1f4c1f29b782968ff1/pkg/controller/certificates/signer/cfssl_signer.go)\nthat signs all approved certificates with a local key. This key is generally\nconfigured to be the key of the cluster's root certificate authority, but is not\nrequired to be.\n\nThe GCP controller manager implements a\n[signer](https://github.com/kubernetes/cloud-provider-gcp/blob/08fa1e3260ffb267682762e24ba93692000e3be8/cmd/gcp-controller-manager/csr_signer.go)\nthat uses a webhook to sign all approved CSRs. This allows the root certificate\nauthority secret material to be stored and maintained outside of the Kubernetes\ncontrol plane.\n\n## Graduation Criteria\n\n### Beta -\u003e GA Graduation\n\nThings to resolve for v1.\n1. .spec.signerName should be non-defaulted and required\n2. Should we disallow the legacy .spec.signerName in v1?\n3. Define how signers indicate terminal failure in signing on a CSR. Fix status conditions perhaps?\n\n## Implementation History\n\n- 1.4: The Certificates API was merged as Alpha\n- 1.6: The Certificates API was promoted to Beta\n- 2020-01-15: Multi-signer design added\n- 2020-01-21: status.certificate field format validation added\n"
+  },
+  {
+    "id": "da3acafed700276c0df2f2615819c783",
+    "title": "External credential providers",
+    "authors": ["@awly"],
+    "owningSig": "sig-auth",
+    "participatingSigs": ["sig-cli", "sig-api-machinery"],
+    "reviewers": ["@liggitt", "@mikedanese"],
+    "approvers": ["@liggitt", "@mikedanese"],
+    "editor": "",
+    "creationDate": "2019-07-11",
+    "lastUpdated": "2019-07-11",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# External credential providers\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Provider configuration](#provider-configuration)\n  - [Provider input format](#provider-input-format)\n  - [Provider output format](#provider-output-format)\n  - [Risks and Mitigations](#risks-and-mitigations)\n    - [Client authentication to the binary](#client-authentication-to-the-binary)\n    - [Invalid credentials before cache expiry](#invalid-credentials-before-cache-expiry)\n  - [Graduation Criteria](#graduation-criteria)\n    - [Beta](#beta)\n- [Alternatives](#alternatives)\n  - [RPC vs exec](#rpc-vs-exec)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nExternal credential providers allow out-of-tree implementation of obtaining\nclient authentication credentials. These providers handle environment-specific\nprovisioning of credentials (such as bearer tokens or TLS client certificates)\nand expose them to the client.\n\n## Motivation\n\nClient authentication credentials for Kubernetes clients are usually specified\nas fields in `kubeconfig` files. These credentials are static and must be\nprovisioned in advance.\n\nThis creates 3 problems:\n\n1. Credential rotation requires client process restart and extra tooling.\n1. Credentials must exist in a plaintext, file on disk.\n1. Credentials must be long-lived.\n\nMany users already use key management/protection systems, such as Key\nManagement Systems (KMS), Trusted Platform Modules (TPM) or Hardware Security\nModules (HSM). Others might use authentication providers based on short-lived\ntokens.\n\nStandard Kubernetes client authentication libraries should support these\nsystems to help with key rotation and protect against key exfiltration.\n\n### Goals\n\n1. Credential rotation without client restart.\n1. Support standard key management solutions.\n1. Support standard token-based protocols.\n1. Provisioning logic lives outside of Kubernetes codebase.\n1. Kubernetes interface is vendor-neutral.\n\n### Non-Goals\n\n1. Exfiltration protection built into Kubernetes.\n1. Kubernetes triggering rotation.\n1. Deprecation of existing authentication options.\n\n## Proposal\n\nA new authentication flow in libraries around `kubeconfig` based on\nexecutables. Before performing a request, client executes a binary and uses its\noutput for authentication.\n\nThere are 2 modes of authentication:\n\n1. bearer tokens\n1. mTLS\n\nProvider response is cached and reused in future requests.\n\nClient is configured with a binary path, optional arguments and environment\nvariables to pass to it.\n\n### Provider configuration\n\nConfiguration is provided via users section of `kubeconfig` file:\n\n```\napiVersion: v1\nkind: Config\nusers:\n- name: my-user\n  user:\n    exec:\n      # API version to use when decoding the ExecCredentials resource. Required.\n      apiVersion: \"client.authentication.k8s.io/\u003cversion\u003e\"\n\n      # Command to execute. Required.\n      command: \"example-client-go-exec-plugin\"\n\n      # Arguments to pass when executing the plugin. Optional.\n      args:\n      - \"arg1\"\n      - \"arg2\"\n\n      # Environment variables to set when executing the plugin. Optional.\n      env:\n      - name: \"FOO\"\n        value: \"bar\"\nclusters:\n- name: my-cluster\n  cluster:\n    server: \"https://1.2.3.4:8080\"\n    certificate-authority: \"/etc/kubernetes/ca.pem\"\ncontexts:\n- name: my-cluster\n  context:\n    cluster: my-cluster\n    user: my-user\ncurrent-context: my-cluster\n```\n\n`apiVersion` specifies the expected version of this API that the plugin\nimplements. If the version differs, client must return an error.\n\n`command` specifies the path to the provider binary. The file at this path must\nbe readable and executable by the client process.\n\n`args` specifies extra arguments passed to the executable.\n\n`env` specifies environment variables to pass to the provider. The environment\nvariables set in the client process are not passed.\n\n### Provider input format\n\n```\n{\n  \"apiVersion\": \"client.authentication.k8s.io/\u003cversion\u003e\",\n  \"kind\": \"ExecCredential\"\n}\n```\n\nProvider can safely ignore `stdin` since input object doesn't carry any data.\n\n### Provider output format\n\n```\n{\n  \"apiVersion\": \"client.authentication.k8s.io/\u003cversion\u003e\",\n  \"kind\": \"ExecCredential\",\n  \"status\": {\n    \"expirationTimestamp\": \"$EXPIRATION\",\n    \"token\": \"$BEARER_TOKEN\",\n    \"clientKeyData\": \"$CLIENT_PRIVATE_KEY\",\n    \"clientCertificateData\": \"$CLIENT_CERTIFICATE\",\n  }\n}\n```\n\n`EXPIRATION` contains the RFC3339 timestamp with credential expiry. Client can\ncache provided credentials until this time.\n\nAfter `EXPIRATION`, client must execute the provider again for any new\nconnections. For `client_key` mode, this applies even if returned certificate\nis still valid.\n\n`BEARER_TOKEN` contains a token for use in `Authorization` header of HTTP\nrequests.\n\n`CLIENT_PRIVATE_KEY` and `CLIENT_CERTIFICATE` contain client TLS credentials in\nPEM format. The certificate must be valid at the time of execution. These\ncredentials are used for mTLS handshakes.\n\n### Risks and Mitigations\n\n#### Client authentication to the binary\n\nCredential provider can authenticate the caller via env vars or arguments\nspecified in its `kubeconfig`. This is optional.\n\nIt is recommended to restrict access to the binary using exec Unix permissions.\n\n#### Invalid credentials before cache expiry\n\nCredentials may become invalid (e.g. expire) after being returned by the\nprovider but before `expirationTimestamp` in the returned `ExecCredential`.\n\nCredential provider should ensure validity of the credentials it returns and\nreturn an error if it can't provide valid credentials.\n\nIn case client gets `401 Unauthorized` or `403 Forbidden` response status from\nremote endpoint when using credentials from a provider, client should\nre-execute the provider, ignoring `expirationTimestamp`.\n\n### Graduation Criteria\n\n#### Beta\n\nFeature is already in Beta.\n\n#### Beta -\u003e GA Graduation\n\n- 3 examples of real world usage\n- support for remote TLS handshakes (e.g. TPM/KMS-hosted keys)\n\n## Alternatives\n\n### RPC vs exec\n\nCredential provider could be exposed as a network endpoint. Instead of\nexecuting a binary and passing request/response over `stdin`/`stdout`, client\ncould open a network connection and send request/response over that.\n\nThe downsides of this approach compared to exec model are:\n\n- if credential provider is remote, design for client authentication is\n  required (aka \"chicken-and-egg problem\")\n- credential provider must constantly run, consuming resources; clients refresh\n  their credentials infrequently\n\n## Implementation History\n\n- 2018-01-29: Proposal submitted https://github.com/kubernetes/community/pull/1503\n- 2018-02-28: Alpha implemented https://github.com/kubernetes/kubernetes/pull/59495\n- 2018-06-04: Promoted to Beta https://github.com/kubernetes/kubernetes/pull/64482\n"
+  },
+  {
+    "id": "10839afce84161369b9ce20210c75bd6",
+    "title": "Service Account signing key retrieval",
+    "authors": ["@mikedanese", "@cceckman"],
+    "owningSig": "sig-auth",
+    "participatingSigs": ["sig-auth"],
+    "reviewers": ["@liggitt", "@enj", "@micahhausler", "@ericchiang"],
+    "approvers": ["@liggitt", "@enj", "@micahhausler", "@ericchiang"],
+    "editor": "TBD",
+    "creationDate": "2018-06-26",
+    "lastUpdated": "2019-07-30",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": ["https://github.com/kubernetes/community/pull/2314/"],
+    "supersededBy": null,
+    "markdown": "\n# Service Account signing key retrieval\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThe Kubernetes API server generates (signs) JSON Web Tokens that are meant to\nauthenticate Kubernetes service accounts. The API server can indeed authenticate\nthese tokens, but in general, no other system can: there is no standard way to\nget the public portion of the keypair used to sign KSA tokens. Systems (\"relying\nparties\") that want authenticate KSA tokens must either send them back to the\nAPI server (via `TokenReview`) or use some provider-specific method to get the\nauthentication key.\n\nIf a relying party could obtain trusted metadata about the service account token\nprovider - in particular, the issuer (`iss`) value and the public key(s) used -\nthen the relying party could authenticate tokens without putting proportionate\nload on the API server. This would allow KSA tokens to be used as a general\nauthentication mechanism, including to services outside the cluster or in other\nclusters.\n\n[OpenID Connect](https://openid.net/connect/) defines a\n[discovery](https://openid.net/specs/openid-connect-discovery-1_0.html)\nmechanism that, given an issuer URL, allows a client to discover the rest of the\nissuer metadata, including the key set. Providing an OIDC-compatible discovery\ndocument would allow flexibility in how relying parties authenticate KSA tokens;\nthey can use existing OIDC authenticators in their language/framework of choice,\nwithout Kuberentes-specific or provider-specific logic.\n\n## Motivation\n\nKubernetes workloads can consume a variety of services from a variety of\nproducers. They have a native identity (KSA), presented in a widely-compatible\nformat (JWT); but only an API server can authenticate the KSA token, since only\nthe API server has access to the public key verifying the signature. If services\nwant to authenticate workloads using KSAs, today, the API server must serve\nevery authentication request (i.e. `TokenReview`).\n\nWhen authenticating across clusters, i.e. from within a cluster to somewhere\nelse, credential management must use a separate system. Someone has to provision\nan identity; provision credentials; grant the workload access to the\ncredentials; and consider:\n\n-   Did I delete ephemeral traces of the credential (e.g. files on my local\n    disk)?\n-   How securely is the credential stored? Is the storage system hardened? Are\n    the ACLs restricted?\n-   How much damage can be caused by an exploit of the workload? Could that\n    compromise a credential with an extended validity period?\n-   How often do the keys expire? How often do I rotate them?\n\nIf services (other than the API server could authenticate KSA tokens directly:\n\n-   the API server wouldn't have to scale with the data-plane (i.e.\n    authentication) load; and\n-   workloads could use native credentials to authenticate to services outside\n    of the cluster.\n\n### Goals\n\n-   Allow (authorized) systems to discover the information they need to\n    authenticate KSA tokens.\n-   Attempt compatibility with OIDC: common libraries that authenticate OIDC\n    tokens should be able to authenticate KSA tokens.\n\nAs a stretch goal / consideration:\n\n-   Support authentication when the API server is not directly reachable by the\n    relying party.\n    -   e.g.: a cloud-based service authenticating an API server that doesn't\n        have a public Internet address.\n\nNote that the API server has a very different flow from OIDC with respect to\ngenerating tokens. As such, our goal is OIDC *compatibility*...\n\n### Non-Goals\n\n...but not OIDC *compliance*.\n\nWe aren't trying to: - Make the KSA token process fully compliant with OIDC\nspecifications. - OIDC includes flows for token acquisition, token exchange,\ngetting user data out of tokens, etc. - But we're only interested in the parts\nrelevant to *relying parties*, i.e. token authentication. - We don't need to do\nsomething `REQUIRED` just because the spec says `REQUIRED`; but we may need to\ndo it if relying parties expect that field. - This may mean that our\nimplementation doesn't comply with the spec; e.g. we might skip `token_endpoint`\nin the\n[discovery document](https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderMetadata),\neven without supporting the\n[Implicit Flow](https://openid.net/specs/openid-connect-core-1_0.html#ImplicitFlowAuth). -\nDefine new cryptographic or authentication protocols. - Change the format of KSA\ntokens, how they're generated, or how they're issued.\n\n## Proposal\n\nA non-resource API will be added to the apiserver that will expose an\n[OIDC discovery](https://openid.net/specs/openid-connect-discovery-1_0.html)-\ncompatible document. This document will reflect the issuer value and provide\nsome additional information about the tokens issued.\n\nIn addition, the API server will serve a JWKS consisting of the public keys from\n`--service-account-key-file` and `--service-account-signing-key`, i.e. all the\npublic keys used for valid KSA tokens. The OIDC discovery document will point to\nthe JWKS path as the `jwks_uri`.\n\nFor example, if the API server is configured with the `--service-account-issuer`\nvalue `https://dev.cluster.internal`, the API server could expose the following\nconfiguration:\n\n```\n\u003e GET /.well-known/openid-configuration\n{\n  \"issuer\": \"https://dev.cluster.internal\",\n  \"jwks_uri\": \"https://dev.cluster.internal/serviceaccountkeys/v1\",\n  \"authorization_endpoint\": \"urn:kubernetes:programmatic_authorization\",\n  \"response_types_supported\": [\n    \"id_token\"\n  ],\n  \"subject_types_supported\": [\n    \"public\"\n  ],\n  \"id_token_signing_alg_values_supported\": [\n    \"RS256\",\n    \"ES256\"\n  ],\n  \"claims_supported\": [\n    \"sub\",\n    \"iss\"\n  ]\n}\n\u003e GET /serviceaccountkeys/v1\n{\n  \"keys\": [\n    {\n      \"kty\": \"RSA\",\n      \"alg\": \"RS256\",\n      \"use\": \"sig\",\n      \"kid\": \"ccab4acb107920dc284c96c6205b313270672039\",\n      \"n\": \"wWGfvdCEjJJy7CQpGcTq6GghmqWLi9H4SNHNTtFMfIDPsv-aWj1e_iSO22505BlC9UcL9LvlSyVH8HmQUy5916YNqxCbhPFPabBAv0a-CpVuzbbyhpDNP3RkRIJgxlzPDh_dB11cbPTQ3yz0A0JARX3QNZfIQ8LFiZ1vh0iZAIm-I3eZeI4QZigImNDviZstSoHB2Ny1tsRmpZn-neYZCxYq717buFctnCVvot4iCwcQpeaGdniqYNDxzN4KlQwwDeCVJm-K0rG9nkiqZ_rq8SgCxi_l7NyF2ZURNTTzZyDwYfBR7jZUhbmjxIDoDZalsa1Tzzy1vzqBfxkFD5Z03w\",\n      \"e\": \"AQAB\"\n    },\n    {\n      \"kty\": \"RSA\",\n      \"alg\": \"RS256\",\n      \"use\": \"sig\",\n      \"kid\": \"8770f6158b125040b98e50a1e0e6790ff2f9ea09\",\n      \"n\": \"vpgsIIPqDO3A3dEuRCIZvQinyfME0BjH_RbeyLAAvrQx-Sv08ryFPjplqxm5t9mC0yULrhOmaIZCVfIuYn5n_dOblZNhpIpoy89bP0qNwV7gxsNv-0Tdu9nj4ymxeoaby6SFiv_c8P2JZ0CSqif_qXgj-o0TqU20FEv1hkizzQWDzFsKZ__IABAkdKfpGqQTOBTylFG9HFLV1tdh9AAdhVRf40982rksaOSDWvN_sfxiz6midGPgG0OOnMnwKAW-3BBNNd_uUrD9baSXZPFA8zo9dlkhQhfrFgg_U6ke4M5DPyFiPKOVitBzpL1Kth_patVZvnBGXtq2frbReF-6pw\",\n      \"e\": \"AQAB\"\n    },\n    {\n      \"kty\": \"RSA\",\n      \"alg\": \"RS256\",\n      \"use\": \"sig\",\n      \"kid\": \"68241231bbf0df8f9123d018cf9e601e2aa3673a\",\n      \"n\": \"rHozcxeim9flTWQxqC1ObpGP0EjpkUHVHpHNX8WGHHnMcVi63_9PaHn2cJeFuPF9qkI1dMPXeoX0m33N0tgM9-KOSmTg1oGbyJGoUYMFI-A7tdxoobb91LGjeNWJC0la0gLOGPcQ6zQLEU5RGftCZT0wElxMuwEH7FZoVBn5i8Ddvc2ADd4bFW0f_FckwFYN1rIU1uLf6coku_1xBfae3b_JiBq38QOGXPdPgxfPzmJEvIz_LB2WOIcwhl97DY32BQU7l_lNLYz6wMg9HeCKolypPIFEGNxLj1TcuOhwP5-BnSja-PvrB-1FN1JyzlL3nh__uJv8SoKPn0CoBPueWw\",\n      \"e\": \"AQAB\"\n    }\n  ]\n}\n```\n\nThe API server would treat these as `nonResourceURLs`, and restrict access\nappropriately. We will consider expanding `system:public-info-viewer` RBAC\nClusterRole to grant access to the new paths; some other roles may have\npermission already via a pattern match on `nonResourceURLs` (e.g.\n`cluster-admin`).\n\n## Implementation History\n\n-   2018-06-26: Proposed in https://github.com/kubernetes/community/pull/2314\n-   2018, 2019: Various comments on pull request\n-   2019-07-30: Moved to a KEP (with no edits from the original proposal)\n-   2019-08-05: Updated KEP with more details.\n-   2019-10-18: Updated KEP with more RBAC details.\n"
+  },
+  {
+    "id": "713a08753d52d201e91d1f3c36f7a654",
+    "title": "Bound Service Account Tokens",
+    "authors": ["@mikedanese"],
+    "owningSig": "sig-auth",
+    "participatingSigs": null,
+    "reviewers": null,
+    "approvers": ["@liggitt", "TBD"],
+    "editor": "",
+    "creationDate": "2019-08-06",
+    "lastUpdated": "2019-08-06",
+    "status": "implemented",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Bound Service Account Tokens\n\n## Table Of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Background](#background)\n- [Motivation](#motivation)\n- [Design Details](#design-details)\n  - [Token attenuations](#token-attenuations)\n    - [Audience binding](#audience-binding)\n    - [Time binding](#time-binding)\n    - [Object binding](#object-binding)\n  - [API Changes](#api-changes)\n    - [Add \u003ccode\u003etokenrequests.authentication.k8s.io\u003c/code\u003e](#add-)\n    - [Modify \u003ccode\u003etokenreviews.authentication.k8s.io\u003c/code\u003e](#modify-)\n    - [Example Flow](#example-flow)\n  - [Service Account Authenticator Modification](#service-account-authenticator-modification)\n  - [ACLs for TokenRequest](#acls-for-tokenrequest)\n  - [Graduation Criteria](#graduation-criteria)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis KEP describes an API that would allow workloads running on Kubernetes\nto request JSON Web Tokens that are audience, time and eventually key bound.\n\n## Background\n\nKubernetes already provisions JWTs to workloads. This functionality is on by\ndefault and thus widely deployed. The current workload JWT system has serious\nissues:\n\n1.  Security: JWTs are not audience bound. Any recipient of a JWT can masquerade\n    as the presenter to anyone else.\n1.  Security: The current model of storing the service account token in a Secret\n    and delivering it to nodes results in a broad attack surface for the\n    Kubernetes control plane when powerful components are run - giving a service\n    account a permission means that any component that can see that service\n    account's secrets is at least as powerful as the component.\n1.  Security: JWTs are not time bound. A JWT compromised via 1 or 2, is valid\n    for as long as the service account exists. This may be mitigated with\n    service account signing key rotation but is not supported by client-go and\n    not automated by the control plane and thus is not widely deployed.\n1.  Scalability: JWTs require a Kubernetes secret per service account.\n\n## Motivation\n\nWe would like to introduce a new mechanism for provisioning Kubernetes service\naccount tokens that is compatible with our current security and scalability\nrequirements.\n\n## Design Details\n\nInfrastructure to support on demand token requests will be implemented in the\ncore apiserver. Once this API exists, a client of the apiserver will request an\nattenuated token for its own use. The API will enforce required attenuations,\ne.g. audience and time binding.\n\n### Token attenuations\n\n#### Audience binding\n\nTokens issued from this API will be audience bound. Audience of requested tokens\nwill be bound by the `aud` claim. The `aud` claim is an array of strings\n(usually URLs) that correspond to the intended audience of the token. A\nrecipient of a token is responsible for verifying that it identifies as one of\nthe values in the audience claim, and should otherwise reject the token. The\nTokenReview API will support this validation.\n\n#### Time binding\n\nTokens issued from this API will be time bound. Time validity of these tokens\nwill be claimed in the following fields:\n\n* `exp`: expiration time\n* `nbf`: not before\n* `iat`: issued at\n\nA recipient of a token should verify that the token is valid at the time that\nthe token is presented, and should otherwise reject the token. The TokenReview\nAPI will support this validation.\n\nCluster administrators will be able to configure the maximum validity duration\nfor expiring tokens. During the migration off of the old service account tokens,\nclients of this API may request tokens that are valid for many years. These\ntokens will be drop in replacements for the current service account tokens.\n\n#### Object binding\n\nTokens issued from this API may be bound to a Kubernetes object in the same\nnamespace as the service account. The name, group, version, kind and uid of the\nobject will be embedded as claims in the issued token. A token bound to an\nobject will only be valid for as long as that object exists.\n\nOnly a subset of object kinds will support object binding. Initially the only\nkinds that will be supported are:\n\n* v1/Pod\n* v1/Secret\n\nThe TokenRequest API will validate this binding.\n\n### API Changes\n\n#### Add `tokenrequests.authentication.k8s.io`\n\nWe will add an imperative API (a la TokenReview) to the\n`authentication.k8s.io` API group:\n\n```golang\ntype TokenRequest struct {\n  Spec   TokenRequestSpec\n  Status TokenRequestStatus\n}\n\ntype TokenRequestSpec struct {\n  // Audiences are the intendend audiences of the token. A token issued\n  // for multiple audiences may be used to authenticate against any of\n  // the audiences listed. This implies a high degree of trust between\n  // the target audiences.\n  Audiences []string\n\n  // ValidityDuration is the requested duration of validity of the request. The\n  // token issuer may return a token with a different validity duration so a\n  // client needs to check the 'expiration' field in a response.\n  ValidityDuration metav1.Duration\n\n  // BoundObjectRef is a reference to an object that the token will be bound to.\n  // The token will only be valid for as long as the bound object exists.\n  BoundObjectRef *BoundObjectReference\n}\n\ntype BoundObjectReference struct {\n  // Kind of the referent. Valid kinds are 'Pod' and 'Secret'.\n  Kind string\n  // API version of the referent.\n  APIVersion string\n\n  // Name of the referent.\n  Name string\n  // UID of the referent.\n  UID types.UID\n}\n\ntype TokenRequestStatus struct {\n  // Token is the token data\n  Token string\n\n  // Expiration is the time of expiration of the returned token. Empty means the\n  // token does not expire.\n  Expiration metav1.Time\n}\n\n```\n\nThis API will be exposed as a subresource under a serviceaccount object. A\nrequestor for a token for a specific service account will `POST` a\n`TokenRequest` to the `/token` subresource of that serviceaccount object.\n\n#### Modify `tokenreviews.authentication.k8s.io`\n\nThe TokenReview API will be extended to support passing an additional audience\nfield which the service account authenticator will validate.\n\n```golang\ntype TokenReviewSpec struct {\n  // Token is the opaque bearer token.\n  Token string\n  // Audiences is the identifier that the client identifies as.\n  Audiences []string\n}\n```\n\n#### Example Flow\n\n```\n\u003e POST /apis/v1/namespaces/default/serviceaccounts/default/token\n\u003e {\n\u003e   \"kind\": \"TokenRequest\",\n\u003e   \"apiVersion\": \"authentication.k8s.io/v1\",\n\u003e   \"spec\": {\n\u003e     \"audience\": [\n\u003e       \"https://kubernetes.default.svc\"\n\u003e     ],\n\u003e     \"validityDuration\": \"99999h\",\n\u003e     \"boundObjectRef\": {\n\u003e       \"kind\": \"Pod\",\n\u003e       \"apiVersion\": \"v1\",\n\u003e       \"name\": \"pod-foo-346acf\"\n\u003e     }\n\u003e   }\n\u003e }\n{\n  \"kind\": \"TokenRequest\",\n  \"apiVersion\": \"authentication.k8s.io/v1\",\n  \"spec\": {\n    \"audience\": [\n      \"https://kubernetes.default.svc\"\n    ],\n    \"validityDuration\": \"99999h\",\n    \"boundObjectRef\": {\n      \"kind\": \"Pod\",\n      \"apiVersion\": \"v1\",\n      \"name\": \"pod-foo-346acf\"\n    }\n  },\n  \"status\": {\n    \"token\":\n    \"eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJz[payload omitted].EkN-[signature omitted]\",\n    \"expiration\": \"Jan 24 16:36:00 PST 3018\"\n  }\n}\n```\n\nThe token payload will be:\n\n```\n{\n  \"iss\": \"https://example.com/some/path\",\n  \"sub\": \"system:serviceaccount:default:default,\n  \"aud\": [\n    \"https://kubernetes.default.svc\"\n  ],\n  \"exp\": 24412841114,\n  \"iat\": 1516841043,\n  \"nbf\": 1516841043,\n  \"kubernetes.io\": {\n    \"serviceAccountUID\": \"c0c98eab-0168-11e8-92e5-42010af00002\",\n    \"boundObjectRef\": {\n      \"kind\": \"Pod\",\n      \"apiVersion\": \"v1\",\n      \"uid\": \"a4bb8aa4-0168-11e8-92e5-42010af00002\",\n      \"name\": \"pod-foo-346acf\"\n    }\n  }\n}\n```\n\n### Service Account Authenticator Modification\n\nThe service account token authenticator will be extended to support validation\nof time and audience binding claims.\n\n### ACLs for TokenRequest\n\nThe NodeAuthorizer will allow the kubelet to use its credentials to request a\nservice account token on behalf of pods running on that node. The\nNodeRestriction admission controller will require that these tokens are pod\nbound.\n\n### Graduation Criteria\n\n#### Beta -\u003e GA Graduation\n\n- TBD\n"
+  },
+  {
+    "id": "e47d7f3d5f50b51df5fa6c8963958b49",
+    "title": "Extended NodeRestrictions for Pods",
+    "authors": ["tallclair"],
+    "owningSig": "sig-auth",
+    "participatingSigs": ["sig-node", "sig-cluster-lifecycle"],
+    "reviewers": ["derekwaynecarr", "neolit123", "deads2k"],
+    "approvers": ["liggitt", "derekwaynecarr", "neolit123", "deads2k"],
+    "editor": "TBD",
+    "creationDate": "2019-09-16",
+    "lastUpdated": "",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Extended NodeRestrictions for Pods\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Background](#background)\n  - [Threat Model](#threat-model)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Label Restrictions](#label-restrictions)\n  - [PodStatus Restrictions](#podstatus-restrictions)\n  - [OwnerReferences](#ownerreferences)\n  - [Risks and Mitigations](#risks-and-mitigations)\n    - [Breaking Services](#breaking-services)\n    - [Namespace Annotation Policy](#namespace-annotation-policy)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n  - [Munge Mirror Pods](#munge-mirror-pods)\n  - [MVP mitigation of known threats](#mvp-mitigation-of-known-threats)\n  - [Restrict namespaces](#restrict-namespaces)\n  - [Weaker label restrictions](#weaker-label-restrictions)\n  - [Annotation Restrictions](#annotation-restrictions)\n  - [Alternative Label Modifications](#alternative-label-modifications)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n**ACTION REQUIRED:** In order to merge code into a release, there must be an issue in\n[kubernetes/enhancements] referencing this KEP and targeting a release milestone **before\n[Enhancement Freeze](https://github.com/kubernetes/sig-release/tree/master/releases) of the targeted\nrelease**.\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e.,\n[kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be\nupdated for the enhancement to be released.\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link\n      to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to\n      [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list\n      discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked\n`implementable` should be approved by each of the KEP approvers. If any of those approvers is no\nlonger appropriate than changes to that list should be approved by the remaining approvers and/or\nthe owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement\nis being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nExtend the [NodeRestriction][] admission controller to add further limitations on a node's effect on\npods:\n\n1. Restrict labels to per-namespace whitelisted keys\n2. Prevent nodes from modifying labels through a `pod/status` update.\n3. Restrict mirror pod OwnerReferences to only allow a node reference.\n\n## Motivation\n\nThe [Node Authorizer][] and associated [NodeRestriction][] controllers introduced the concept of\nnode isolation: making it possible to prevent a compromised node from compromising the whole\ncluster. A key step is limiting the node's access to resources to only those required by pods\nrunning on the node. For example, a node can only read secrets referenced by pods on that node.\n\nHowever, there are other controllers in the cluster interacting with pods on a node, and under some\ncircumstances those controllers can be manipulated to do attacker's will. This is known as a\n[\"confused deputy attack\"](https://en.wikipedia.org/wiki/Confused_deputy_problem).\n\nExamples include:\n\n- Making a pod match a service selector in order to [man-in-the-middle][] (MITM) the service\n  traffic.\n- Making a pod match a {ReplicaSet,StatefulSet,etc.} controller so the controller deletes\n  legitimate replicas, thereby DoSing the application.\n\nThere are likely other 3rd party controllers that could also be manipulated.\n\nIn order to mitigate these attack scenarios, the node (and all pods running on the node) must not be\nable to manipulate pods in a way that they're matched by controllers.\n\n### Background\n\nThe Kubelet has two mechanisms that can be used to manipulate pods. The first is through the\npod/status subresource. Despite the name, an update to pod/status can also update (some) of the\npod's metadata. Of particular interest are updates to labels and annotations. Note that pod/status\nupdates _cannot_ modify the OwnerReferences [[1][]].\n\nThe second mechanism is through the creation of \"mirror pods\". The Kubelet can run pods from other\nsources than the API server, such as a static manifest directory or pulled from an HTTP\nserver. These pods are referred to as \"static pods\". Since static pods don't come from the API\nserver, components that read pods from the API wouldn't know about them. To compensate, the Kubelet\ncreates a \"mirror pod\", which reflects the state of the static pod it's running.\n\nMirror pods have some special properties. They are identified with a special annotation,\n`kubernetes.io/config.mirror`, and the Kubelet is only authorized to create mirror pods, and only on\nthe same node (itself). The Kubelet won't run a mirror pod (since it's actually running a static\npod). Mirror pods are also restricted from using a service account, or secrets, configmaps,\npersistent volumes, and other resources restricted by the node authorizer, in order to prevent an\nattacker from bypassing the authorization by creating mirror pods.\n\n[1]: https://github.com/kubernetes/kubernetes/blob/ab73a018de51bddf9d03d6fed6e867b60196c796/pkg/registry/core/pod/strategy.go#L162-L171\n\n[Node Authorizer]: https://kubernetes.io/docs/reference/access-authn-authz/node/\n[NodeRestriction]: https://kubernetes.io/docs/reference/access-authn-authz/node/\n[man-in-the-middle]: https://en.wikipedia.org/wiki/Man-in-the-middle_attack\n\n### Threat Model\n\nAt a high level, this proposal targets clusters making effective use of node isolation to separate\nsensitive workloads or limit the blast radius of a successful node compromise. More specifically, it\nmakes the following assumptions:\n\n- The cluster uses scheduling constraints such as node selectors \u0026 taints / tolerations to separate\n  workloads of different trust or privilege levels.\n- An attacker has compromised a low-privilege node meaning they have code execution as root in the\n  host namespaces (i.e. a container escape). Low-privilege in this case means the node is not\n  hosting any privileged workloads with permissions to trivially take over the cluster\n  (e.g. unrestricted pod creation, read secrets, etc).\n\n### Goals\n\n- Prevent a compromised node from manipulating controllers to execute confused deputy attacks.\n\n### Non-Goals\n\n- Solving all node isolation issues.\n- Solving all possible man-in-the-middle attacks.\n\n## Proposal\n\nAll restrictions will be enforced through the [NodeRestriction][] admission controller. These\nextensions will be guarded by the `MirrorPodNodeRestriction` feature gate.\n\nNodes are not granted the `update` or `patch` permissions on pods, but may update `pod/status`. All\nlabel and owner reference updates will be forbidden through `pod/status` updates, so restrictions\nwill not be checked on status updates. In other words, a node _can_ update the status on a pod that\nhas un-whitelisted labels.\n\n### Label Restrictions\n\nA new reserved annotation will be introduced for namespaces to whitelist mirror-pod labels:\n\n```\nnode.kubernetes.io/mirror.allowed-label-keys = \"key1,key2,...\"\n```\n\nWhen the NodeRestriction controller receives a mirror pod create a request for a node, it will check\nthe pod for labels. If it is labeled, the pod's namespace is checked for the allowed-label-keys\nannotation. If any of the mirror-pod's labels are not whitelisted, or the annotation is absent, the\ncreate request will be rejected.\n\nThe Kubelet does not currently label pods, nor are there official label keys that apply to\npods. However, there are a few labels that are commonly applied to system addons \u0026 static pods:\n\n- `component` (used by [kubeadm][kubeadm-labels])\n- `tier` (used by [kubeadm][kubeadm-labels])\n- `k8s-app` (common on [addons][addons-k8s-app])\n\nThe `k8s-app` label is used to match controllers for system components, and therefore should be\nexplicitly disallowed.\n\n`kubeadm` should be modified to whitelist the `component` and `tier` labels, or potentially drop\nthem if they're not required.\n\n[kubeadm-labels]: https://github.com/kubernetes/kubernetes/blob/e682310dcc5d805a408e0073e251d99b8fe5c06d/cmd/kubeadm/app/util/staticpod/utils.go#L60\n[addons-k8s-app]: https://github.com/kubernetes/kubernetes/blob/e682310dcc5d805a408e0073e251d99b8fe5c06d/cluster/addons/kube-proxy/kube-proxy-ds.yaml#L23\n\n### PodStatus Restrictions\n\nSome metadata can be modified through a `pod/status` subresource update. OwnerRefrences are\nrestricted from pod/status updates, but labels \u0026 annotations can be updated. Going forward, nodes\nwill be restricted from making any label changes through `pod/status` updates.\n\nAs the kubelet doesn't make any label updates through this request, this change will be rolled out\nwithout a feature gate in **v1.17**.\n\n### OwnerReferences\n\nOwnerReferences can be set on mirror pods today. With the new restrictions, mirror pods are only\nallowed a single owner reference (or none), and it must refer to the node:\n\n```go\n metav1.OwnerReference{\n  APIVersion: \"v1\"\n  Kind: \"Node\"\n  Name: node.Name\n  UID:  node.UID\n  Controller: true  // Prevent other controllers from adopting the pod.\n  BlockOwnerDeletion: false\n}\n```\n\nThe Kubelet will start injecting this OwnerReference into mirror-pods in **v1.17**, unguarded by a\nfeature gate.\n\nThe node owner reference will eventually be required, but due to apiserver-node version skew, this\nmust happen at least 2 releases after nodes start injecting this OwnerReference.\n\n### Risks and Mitigations\n\n#### Breaking Services\n\nSome Kubernetes setups depend on statically serving services today. Applying these mitigations will\nlikely break these clusters. There is no way to apply these changes in a fully backwards compatible\nway, so users or operators of such clusters will be required to whitelist the required labels prior\nto enabling the `MirrorPodNodeRestriction` feature gate (or upgrading to a release with the feature\ngate enabled).\n\nHere is a kubectl monstrosity for listing the labels that need to be whitelisted on each namespcae:\n\n```\nkubectl get pods --all-namespaces -o=go-template='{{range .items}}{{if .metadata.annotations}}{{if (index .metadata.annotations \"kubernetes.io/config.mirror\") }}{{$ns := .metadata.namespace}}{{range $key, $value := .metadata.labels}}{{$ns}}{{\": \"}}{{$key}}{{\"\\n\"}}{{end}}{{end}}{{end}}{{end}}' | sort -u\n```\n\nThis command gives output as `namespace: label-key` pairs, for example:\n\n```\n$ kubectl get pods --all-namespaces -o=go-template='{{range .items}}{{if .metadata.annotations}}{{if (index .metadata.annotations \"kubernetes.io/config.mirror\") }}{{$ns := .metadata.namespace}}{{range $key, $value := .metadata.labels}}{{$ns}}{{\": \"}}{{$key}}{{\"\\n\"}}{{end}}{{end}}{{end}}{{end}}' | sort -u\nkube-system: component\nkube-system: extra\nkube-system: tier\n```\n\nWhich should be translated into an annotation on the kube-system namespace:\n\n```\nnode.kubernetes.io/mirror.allowed-label-keys: \"component,extra,tier\"\n```\n\nLike so:\n\n```\n$ kubectl annotate namespaces kube-system node.kubernetes.io/mirror.allowed-label-keys=\"component,extra,tier\"\n```\n\n#### Namespace Annotation Policy\n\nThere is prior art for representing namespaced policy through annotations, such as with the\n[PodNodeSelector][] or [PodTolerationRestriction][] admission controllers. However, this is a model\nwe're trying to move away from for general namespaced-policy specification, so adding another\nnamespace annotation is considered a risk.\n\nIn this case, I think an exception is warranted given that:\n\n1. Use of the annotation is very niche\n2. The annotation is narrowly-scoped. Even when required, it will probably only be needed on a small\n   number (typically 1) of namespaces.\n3. The annotation is expected to be static. New namespaces won't need to be annotated, and the\n   annotation value is tied to the cluster setup.\n\n[PodNodeSelector]: https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#configuration-annotation-format\n[PodTolerationRestriction]: https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#podtolerationrestriction\n\n## Design Details\n\n### Test Plan\n\nOur CI environment does not depend on static pods serving services, so we can enable the feature\ngate in the standard Kubernetes E2E environment. The restrictions can be verified by impersonating a\nnode's identity and ensuring illegal mirror pods cannot be created while legal pods can be.\n\nTo test the Kubelet modifications, a privileged pod can write a static pod manifest to a node and\nverify the expected changes are made.\n\n### Graduation Criteria\n\nThe feature gate will initially be in a default-disabled alpha state. Graduating to beta will make\nthe feature enabled by default, and will break users who have not taken steps to whitelist\nmirror-pod labels.\n\nHere is an approximate graduation schedule, specific release numbers subject to change:\n\n- v1.17\n  - MirrorPodNodeRestriction: **alpha**\n- v1.19\n  - MirrorPodNodeRestriction: **beta**\n- v1.20\n  - MirrorPodNodeRestriction: **GA**\n\n### Upgrade / Downgrade Strategy\n\nUpgrade / downgrade is only meaningful for enabling / disabling the feature gate. If no explicit\naction is taken, this will happen on upgrade when the feature graduates to beta.\n\nEnabling the feature will not affect pods that are already running. If a new static pod is deployed,\nor a node needs to (re)create a static pod with an illegal label, that mirror pod will be\nrejected. The Kubelet will still run the pod locally, but it will not be exposed through the\nKubernetes API, controllers won't be able to find it, and the scheduler may not account for its\nresources.\n\nRolling back / disabling the feature will not affect existing pods. If a mirror pod was previously\nrejected, the Kubelet will attempt to recreate it and it will now be allowed.\n\n## Implementation History\n\n- 2019-09-16 - KEP proposed\n\n## Alternatives\n\n### Munge Mirror Pods\n\nAn alternative to outright rejecting invalid mirror pods, the NodeRestriction controller could\nmodify the mirror pods to conform to the restrictions. For example, the controller could:\n\n- Remove illegal labels, and dump them into an annotation for audit purposes\n  (e.g. `kubernetes.io/config.removed-labels`)\n- Remove illegal owner references, also dumping them into an annotation\n- Add the node owner reference (and require it)\n\nA problem with this approach is that the Kubelet will not attempt to recreate a mirror pod with\nillegal labels once the labels are whitelisted. In contrast, if the pod is outright rejected, then\nas soon as its labels are whitelisted the Kubelet would try to recreate it and succeed. This\nargument does not apply to the owner references, but the benefit of only modifying the owner\nreference is weaker.\n\n### MVP mitigation of known threats\n\nAn MVP of this proposal to mitigate the [2 motivating examples](#motivation) must include:\n\n1. Prevent nodes from modifying arbitrary labels through `pod/status` updates.\n2. Prevent nodes from setting arbitrary labels on mirror pods.\n3. Prevent nodes from setting arbitrary owner references on mirror pods.\n\nAn MVP would exclude the speculative annotation restrictions. It could optionally take a blacklist\napproach to label restrictions rather than a whitelist approach, but doing so would force every\nservice label to use the `node-restriction.kubernetes.io/` prefix to prevent the MITM threat.\n\n### Restrict namespaces\n\nFor additional defense-in-depth, mirror pods could be restricted to whitelisted namespaces, for\nexample only namespaces with a `node.kubernetes.io/mirror.allowed` annotation. We may consider\nthis change in a future release.\n\n### Weaker label restrictions\n\nAlternatives to the whitelist restriction approach were considered.\n\n**Whitelist prefix**\n\nThe original version of this proposal suggested using a `unrestricted.node.kubernetes.io/*` prefix\nfor whitelisted labels. This approach requires migrating static pods to a new label through a\ncomplicated rollout procedure coordinated between services and multiple feature gates.\n\n**Explicitly opt mirror pods out controllers**\n\nRequires the controllers to check for the `kubernetes.io/config.mirror` annotation before matching a\npod. While we could make this change for internal controllers, there is no way to enforce it for\nthird-party controllers, so this approach would be less-safe. It also still requires the\n`pod/status` label update restriction and owner ref restrictions to be complete.\n\n**Blacklist labels**\n\nRather than forbidding all labels except those under `unrestricted.node.kubernetes.io/`, we could\n_allow_ all labels except those under `restricted.node.kubernetes.io/`.\n\nThis change is more consistent with the [self-labeling restrictions on\nnodes](0000-20170814-bounding-self-labeling-kubelets.md), but has a much broader impact. Pod labels\n\u0026 label selectors are much more widely used than node labels, and less cluster dependent. This means\nthe labels are often set through third party deployment tools, such as helm. In order to safely\nmatch labels, ALL labels consumed by controllers or other security-sensitive operations would need\nto be moved to the blacklisted domain. Doing so would be a disruptive change, and force all labels\nto be under the same domain.\n\n**Whitelist configuration**\n\nWe could provide a configurable option (flag / ComponentConfig) to the NodeRestriction admission\ncontroller to explicitly whitelist specific labels. This would be in addition to the\n`unrestricted.node.kubernetes.io/` prefix. Alternatively, it could optionally include prefixes, and\nwe could make `\"unrestricted.node.kubernetes.io/*\"` be the default value of the option.\n\nProviding this option is tempting, but it increases the configurable surface area with a\nsecurity-sensitive option that is easy to misunderstand. For example, a system service matching\nmirror pods should explicitly opt-in to using the insecure labels to make the implications\nexplicit. If any labels can be whitelisted, it becomes harder to audit the cluster.\n\n### Annotation Restrictions\n\nIn addition to label \u0026 owner restrictions, annotation keys could be restricted too. I am still open\nto adding these restrictions in a future extension, but doing so is contingent on concrete use\ncases.\n\nUnder these restrictions, the Kubelet would be prevented from updating pod annotations or creating\nmirror pods with annotations, except for whitelisted keys:\n\n1. Any annotations starting with `unrestricted.node.kubernetes.io/` are allowed.\n2. Annotations the Kubelet currently uses are allowed:\n- `ConfigMirrorAnnotationKey = \"kubernetes.io/config.mirror\"`\n- `ConfigHashAnnotationKey = \"kubernetes.io/config.hash\"`\n- `ConfigFirstSeenAnnotationKey = \"kubernetes.io/config.seen\"`\n- `ConfigSourceAnnotationKey = \"kubernetes.io/config.source\"`\n3. Well-known annotations that may be used on static pods are allowed:\n- `PodPresetOptOutAnnotationKey = \"podpreset.admission.kubernetes.io/exclude\"\n- `SeccompPodAnnotationKey = \"seccomp.security.alpha.kubernetes.io/pod\"`\n- `SeccompContainerAnnotationKeyPrefix = \"container.seccomp.security.alpha.kubernetes.io/\"` (prefix\n  match)\n- `ContainerAnnotationKeyPrefix = \"container.apparmor.security.beta.kubernetes.io/\"` (prefix match)\n- `PreferAvoidPodsAnnotationKey = \"scheduler.alpha.kubernetes.io/preferAvoidPods\"`\n- `BootstrapCheckpointAnnotationKey = \"node.kubernetes.io/bootstrap-checkpoint\"`\n\n### Alternative Label Modifications\n\nSeveral alternative label modification schemes were discussed, including:\n\n- Out right rejecting pods with illegal labels\n- Munging the labels to fit the allowed schema\n\nFor more details, see https://github.com/kubernetes/enhancements/pull/1243#issuecomment-540758654\n"
+  },
+  {
+    "id": "40aa0f7216cbc1d48d45b29fb2f5ef61",
+    "title": "Enhance HPA Metrics Specificity",
+    "authors": ["@directxman12"],
+    "owningSig": "sig-autoscaling",
+    "participatingSigs": ["sig-instrumentation"],
+    "reviewers": ["@brancz", "@maciekpytel"],
+    "approvers": ["@brancz", "@maciekpytel", "@directxman12"],
+    "editor": "@directxman12",
+    "creationDate": "2018-04-19",
+    "lastUpdated": "2018-04-19",
+    "status": "implemented",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Enhance HPA Metrics Specificity\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThe `External` metric source type in the HPA currently supports passing\na metric label selectors, which is passed to the custom metrics API\n(custom.metrics.k8s.io) to select a more specific metrics series.  This\nallows users to more easily make use of existing metrics structure,\nwithout need to manipulate their metrics labeling and ingestion\nexternally.\n\nAdditionally, it supports the `targetAverageValue` field, which allows\nartificially dividing an external metric by the number of replicas in the\ntarget scalable.\n\nThis proposal brings both of those fields to the `Object` metric source\ntype, and further brings the selector field to the `Pods` metric source\ntype, making both types more flexible and bringing them in line with the\n`External` metrics types.\n\n## Motivation\n\nWith custom-metrics-based autoscaling, users frequently ask how to select\nmore specific metrics in their metric storage.  For instance, a user might\nhave message queue statefulset with several queues producing metrics:\n\n```\nqueue_length{statefulset=\"foo\",pod=\"foo-1\",queue=\"some-jobs\"}\nqueue_length{statefulset=\"foo\",pod=\"foo-1\",queue=\"other-jobs\"}\n```\n\nSuppose they have a pool of works that they wish to scale for each queue.\nIn the current-day HPA, it's non-trivial to allow selecting the metric for\na specific queue.  Current suggestions are metric-backend-specific (for\ninstance, you could create a Prometheus recording rule to relabel or\nrename the metric), and often involve making external changes to the\nmetrics pipeline.\n\nWith the addition of the metrics label selector, users could simply select\nthe queue using the label selector:\n\n```yaml\n- type: Object\n  object:\n    describedObject:\n      kind: StatefulSet\n      apiVersion: apps/v1\n      name: foo\n    target:\n      type: Value\n      value: 2\n    metric:\n      name: queue_length\n      selector: {matchLabels: {queue: some-jobs}}\n```\n\nSimilarly, in discussions of scaling on queues, being able to divide\na target backlog length by the number of available pods is often useful --\nfor instance, a backlog length of 3 might be acceptable if there are three\npods processing items, but not if there is only one.\n\n### Goals\n\n- The autoscaling/v2 API is updated with the additional fields\n  described below.\n- A corresponding change is made to the custom metrics API to support the\n  additional label selector.\n- The testing adapter is updated to support these changes (for e2e\n  purposes).\n\n### Non-Goals\n\nIt is outside of the purview of the KEP to ensure that current custom\nmetrics adapters support the new changes -- this is up to those adapters\nmaintainers.\n\n## Proposal\n\nThe autoscaling/v2 API will be updated the following way:\n\n```go\ntype ObjectMetricSource struct {\n\tDescribedObject CrossVersionObjectReference\n\tTarget MetricTarget\n\tMetric MetricIdentifier\n}\n\ntype PodsMetricSource struct {\n    Target MetricTarget\n\tMetric MetricIdentifier\n}\n\ntype ExternalMetricSouce struct {\n\tMetric MetricIdentifier\n\tTarget MetricTarget\n}\n\ntype ResourceMetricSource struct {\n\tName v1.ResourceName\n\tTarget MetricTarget\n}\n\ntype MetricIdentifier struct {\n\t// name is the name of the given metric\n\tName string\n\t// selector is the selector for the given metric\n\t// +optional\n\tSelector *metav1.LabelSelector\n}\n\ntype MetricTarget struct {\n\tType MetricTargetType // Utilization, Value, AverageValue\n    // value is the raw value of the single metric (valid for object metrics)\n\tValue *resource.Quantity\n\n    // averageValue is the raw value or values averaged across the number\n    // of pods targeted by the HPA (valid for all metric types).\n\tAverageValue *resource.Quantity\n\n    // averageUtilization is the average value (as defined above) as\n    // a percentage of the corresponding average pod request (valid\n    // for resource metrics).\n\tAverageUtilization *int32\n}\n\n// and similarly for the statuses:\n\ntype MetricValueStatus struct {\n    // value is the current value of the metric (as a quantity).\n    // +optional\n    Value *resource.Quantity\n    // averageValue is the current value of the average of the\n    // metric across all relevant pods (as a quantity)\n    // (always reported for resource metrics)\n    // +optional\n    AverageValue *resource.Quantity\n    // currentAverageUtilization is the current value of the average of the\n    // resource metric across all relevant pods, represented as a percentage of\n    // the requested value of the resource for the pods.\n    // +optional\n    AverageUtilization *int32\n}\n\n```\n\nNotice that the `metricName` field is replaced with a new `metric` field,\nwhich encapsulates both the metric name, and an optional label selector,\nwhich takes the form of a standard kubernetes label selector.\n\nThe `targetXXX` fields are replaced by a unified `Target` field that\ncontains the different target types. The `target` field in the Object\nmetric source type is renamed to `describedObject`, since the `target`\nfield is now taken, and to more accurately describe its purpose.\n\nThe `External` source is updated slightly to match the new form of the\n`Pods` and `Object` sources.\n\nThese changes necessitate a second beta of `autoscaling/v2`:\n`autoscaling/v2beta2`.\n\nSimilarly, corresponding changes need to be made to the custom metrics\nAPI:\n\n```go\ntype MetricValue struct {\n\tmetav1.TypeMeta\n\tDescribedObject ObjectReference\n\n\tMetric MetricIdentifier\n\n\tTimestamp metav1.Time\n\tWindowSeconds *int64\n\tValue resource.Quantity\n}\n\ntype MetricIdentifier struct {\n\t// name is the name of the given metric\n\tName string\n\t// selector represents the label selector that could be used to select\n\t// this metric, and will generally just be the selector passed in to\n\t// the query used to fetch this metric.\n\t// +optional\n\tSelector *metav1.LabelSelector\n}\n```\n\nThis will also require bumping the custom metrics API to\n`custom.metrics.k8s.io/v1beta2`.\n\n**Note that if a metrics pipeline works in such a way that multiple series\nare matched by a label selector, it's the metrics adapter's job to deal\nwith it, similarly to the way things current work with the custom metrics\nAPI.**\n\n### Risks and Mitigations\n\nThe main risk around this proposal revolves around metric backend support.\nWhen crafting the initial API, there were two constraints: a) limit\nourselves to an API surface that could be limited in adapters without any\nadditional processing of metrics, and b) avoid creating a new query\nlanguage.\n\nThere are currently three adapter implementations (known to SIG\nAutoscaling): Prometheus, Stackdriver, and Sysdig.  Of those three, both\nPrometheus and Stackdriver map nicely to the `name+labels` abstraction,\nwhile Sysdig does not seem to natively have a concept of labels.  However,\nthis simply means that users of sysdig metrics will not make use of labels\n-- there should be no need for the sysdig adapter to do anything special\nwith the labels besides ignore them.  The \"name+label\" paradigm also seems\nto match nicely with other metric solutions (InfluxDB, DataDog, etc) used\nwith Kubernetes.\n\nAs for moving closer to a query language, this change is still very\nstructured and very limitted.  It requires no additional parsing logic\n(since it uses standard kubernetes label selectors), and translation to\nunderlying APIs and query languages should be relatively simple.\n\n## Graduation Criteria\n\nIn general, we'll want to graduate the autoscaling/v2 and\ncustom.metrics.k8s.io APIs to GA once we have a release with at least one\nadapter up to date, and positive user feedback that does not suggest\nurgent need for further changes.\n\n## Implementation History\n\n- (2018/4/19) Proposal proposed\n- (2018/8/27) Implementation (kubernetes/kubernetes#64097) merged for Kubernetes 1.12\n\n## Alternatives\n\n- Continuing to require out-of-band changes to support more complex metric\n  environments: this induces a lot of friction with traditional\n  Prometheus-style monitoring setups, which favor selecting on labels.\n  Furthermore, the changes required often involve admin intervention,\n  which is not always simple or scalable in larger environments.\n\n- Allow passing full queries instead of metric names: this would make the\n  custom metrics API significantly more scalable, at the cost of adapter\n  complexity, security issues, and lesser portability.  Effectively,\n  adapters would have to implement query rewriting to inject extra labels\n  in to scope metrics down to their target objects, which could in turn\n  cause security issues. Additionally, it makes it a lot hard to port the\n  HPAs between different metrics solutions.\n"
+  },
+  {
+    "id": "fd88cabe6d87b84fb72e4cdba19ccb53",
+    "title": "In-place Update of Pod Resources",
+    "authors": ["@kgolab", "@bskiba", "@schylek", "@vinaykul"],
+    "owningSig": "sig-autoscaling",
+    "participatingSigs": ["sig-node", "sig-scheduling"],
+    "reviewers": ["@bsalamat", "@dashpole", "@derekwaynecarr", "@dchen1107", "@ahg-g", "@k82cn"],
+    "approvers": ["@dchen1107", "@derekwaynecarr", "@ahg-g", "@mwielgus"],
+    "editor": "TBD",
+    "creationDate": "2018-11-06",
+    "lastUpdated": "2018-11-06",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# In-place Update of Pod Resources\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [API Changes](#api-changes)\n    - [Container Resize Policy](#container-resize-policy)\n    - [CRI Changes](#cri-changes)\n  - [Kubelet and API Server Interaction](#kubelet-and-api-server-interaction)\n    - [Kubelet Restart Tolerance](#kubelet-restart-tolerance)\n  - [Scheduler and API Server Interaction](#scheduler-and-api-server-interaction)\n  - [Flow Control](#flow-control)\n    - [Container resource limit update ordering](#container-resource-limit-update-ordering)\n    - [Notes](#notes)\n  - [Affected Components](#affected-components)\n  - [Future Enhancements](#future-enhancements)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis proposal aims at allowing Pod resource requests \u0026 limits to be updated\nin-place, without a need to restart the Pod or its Containers.\n\nThe **core idea** behind the proposal is to make PodSpec mutable with regards to\nResources, denoting **desired** resources. Additionally, PodSpec is extended to\nreflect resources **allocated** to a Pod, and PodStatus is extended to provide\ninformation about **actual** resources applied to the Pod and its Containers.\n\nThis document builds upon [proposal for live and in-place vertical scaling][]\nand [Vertical Resources Scaling in Kubernetes][].\n\n[proposal for live and in-place vertical scaling]:\nhttps://github.com/kubernetes/community/pull/1719\n[Vertical Resources Scaling in Kubernetes]:\nhttps://docs.google.com/document/d/18K-bl1EVsmJ04xeRq9o_vfY2GDgek6B6wmLjXw-kos4\n\n## Motivation\n\nResources allocated to a Pod's Container(s) can require a change for various\nreasons:\n* load handled by the Pod has increased significantly, and current resources\n  are not sufficient,\n* load has decreased significantly, and allocated resources are unused,\n* resources have simply been set improperly.\n\nCurrently, changing resource allocation requires the Pod to be recreated since\nthe PodSpec's Container Resources is immutable.\n\nWhile many stateless workloads are designed to withstand such a disruption,\nsome are more sensitive, especially when using low number of Pod replicas.\n\nMoreover, for stateful or batch workloads, Pod restart is a serious disruption,\nresulting in lower availability or higher cost of running.\n\nAllowing Resources to be changed without recreating the Pod or restarting the\nContainers addresses this issue directly.\n\n### Goals\n\n* Primary: allow to change Pod resource requests \u0026 limits without restarting\n  its Containers.\n* Secondary: allow actors (users, VPA, StatefulSet, JobController) to decide\n  how to proceed if in-place resource resize is not possible.\n* Secondary: allow users to specify which Pods and Containers can be resized\n  without a restart.\n\n### Non-Goals\n\nThe explicit non-goal of this KEP is to avoid controlling full lifecycle of a\nPod which failed in-place resource resizing. This should be handled by actors\nwhich initiated the resizing.\n\nOther identified non-goals are:\n* allow to change Pod QoS class without a restart,\n* to change resources of Init Containers without a restart,\n* eviction of lower priority Pods to facilitate Pod resize,\n* updating extended resources or any other resource types besides CPU, memory.\n\n## Proposal\n\n### API Changes\n\nPodSpec becomes mutable with regards to Container resources requests and\nlimits. PodSpec is extended with information of resources allocated on the\nNode for the Pod. PodStatus is extended to show the actual resources applied\nto the Pod and its Containers.\n\nThanks to the above:\n* Pod.Spec.Containers[i].Resources becomes purely a declaration, denoting the\n  **desired** state of Pod resources,\n* Pod.Spec.Containers[i].ResourcesAllocated (new object, type v1.ResourceList)\n  denotes the Node resources **allocated** to the Pod and its Containers,\n* Pod.Status.ContainerStatuses[i].Resources (new object, type\n  v1.ResourceRequirements) shows the **actual** resources held by the Pod and\n  its Containers.\n\nA new Pod subresource named 'resourceallocation' is introduced to allow\nfine-grained access control that enables Kubelet to set or update resources\nallocated to a Pod, and prevents the user or any other component from changing\nthe allocated resources.\n\n#### Container Resize Policy\n\nTo provide fine-grained user control, PodSpec.Containers is extended with\nResizePolicy map (new object) for each resource type (CPU, memory):\n* NoRestart - the default value; resize Container without restarting it,\n* RestartContainer - restart the Container in-place to apply new resource\n  values. (e.g. Java process needs to change its Xmx flag)\n\nBy using ResizePolicy, user can mark Containers as safe (or unsafe) for\nin-place resource update. Kubelet uses it to determine the required action.\n\nSetting the flag to separately control CPU \u0026 memory is due to an observation\nthat usually CPU can be added/removed without much problem whereas changes to\navailable memory are more probable to require restarts.\n\nIf more than one resource type with different policies are updated, then\nRestartContainer policy takes precedence over NoRestart policy.\n\nAdditionally, if RestartPolicy is 'Never', ResizePolicy should be set to\nNoRestart in order to pass validation.\n\n#### CRI Changes\n\nKubelet calls UpdateContainerResources CRI API which currently takes\n*runtimeapi.LinuxContainerResources* parameter that works for Docker and Kata,\nbut not for Windows. This parameter changes to *runtimeapi.ContainerResources*,\nthat is runtime agnostic, and will contain platform-specific information.\n\n### Kubelet and API Server Interaction\n\nWhen a new Pod is created, Scheduler is responsible for selecting a suitable\nNode that accommodates the Pod.\n\nFor a newly created Pod, Spec.Containers[i].ResourcesAllocated must match\nSpec.Containers[i].Resources.Requests. When Kubelet admits a new Pod, values in\nSpec.Containers[i].ResourcesAllocated are used to determine if there is enough\nroom to admit the Pod. Kubelet does not set Pod's ResourcesAllocated after\nadmitting a new Pod.\n\nWhen a Pod resize is requested, Kubelet attempts to update the resources\nallocated to the Pod and its Containers. Kubelet first checks if the new\ndesired resources can fit the Node allocable resources by computing the sum of\nresources allocated (Pod.Spec.Containers[i].ResourcesAllocated) for all Pods in\nthe Node, except the Pod being resized. For the Pod being resized, it adds the\nnew desired resources (i.e Spec.Containers[i].Resources.Requests) to the sum.\n* If new desired resources fit, Kubelet accepts the resize by updating\n  Pod.Spec.Containers[i].ResourcesAllocated via pods/resourceallocation\n  subresource, and then proceeds to invoke UpdateContainerResources CRI API\n  to update the Container resource limits. Once all Containers are successfully\n  updated, it updates Pod.Status.ContainerStatuses[i].Resources to reflect the\n  new resource values.\n* If new desired resources don't fit, Kubelet rejects the resize, and no\n  further action is taken.\n  - Kubelet retries the Pod resize at a later time.\n\nIf multiple Pods need resizing, they are handled sequentially in the order in\nwhich Pod additions and updates arrive at Kubelet.\n\nScheduler may, in parallel, assign a new Pod to the Node because it uses cached\nPods to compute Node allocable values. If this race condition occurs, Kubelet\nresolves it by rejecting that new Pod if the Node has no room after Pod resize.\n\n#### Kubelet Restart Tolerance\n\nIf Kubelet were to restart amidst handling a Pod resize, then upon restart, all\nPods are admitted at their current Pod.Spec.Containers[i].ResourcesAllocated\nvalues, and resizes are handled after all existing Pods have been added. This\nensures that resizes don't affect previously admitted existing Pods.\n\n### Scheduler and API Server Interaction\n\nScheduler continues to use Pod's Spec.Containers[i].Resources.Requests for\nscheduling new Pods, and continues to watch Pod updates, and updates its cache.\nIt uses the cached Pod's Spec.Containers[i].ResourcesAllocated values to\ncompute the Node resources allocated to Pods. This ensures that it always uses\nthe most recently available resource allocations in making new Pod scheduling\ndecisions.\n\n### Flow Control\n\nThe following steps denote a typical flow of an in-place resize operation for a\nPod with ResizePolicy set to NoRestart for all its Containers.\n\n1. Initiating actor updates Pod's Spec.Containers[i].Resources via PATCH verb.\n1. API Server validates the new Resources. (e.g. Limits are not below\n   Requests, QoS class doesn't change, ResourceQuota not exceeded...)\n1. API Server calls all Admission Controllers to verify the Pod Update.\n   * If any of the Controllers reject the update, API Server responds with an\n     appropriate error message.\n1. API Server updates PodSpec object with the new desired Resources.\n1. Kubelet observes that Pod's Spec.Containers[i].Resources.Requests and\n   Spec.Containers[i].ResourcesAllocated differ. It checks its Node allocable\n   resources to determine if the new desired Resources fit the Node.\n   * _Case 1_: Kubelet finds new desired Resources fit. It accepts the resize\n     and sets Spec.Containers[i].ResourcesAllocated equal to the values of\n     Spec.Containers[i].Resources.Requests by invoking resourceallocation\n     subresource. It then applies the new cgroup limits to the Pod and its\n     Containers, and once successfully done, sets Pod's\n     Status.ContainerStatuses[i].Resources to reflect the desired resources.\n     - If at the same time, a new Pod was assigned to this Node against the\n       capacity taken up by this resource resize, that new Pod is rejected by\n       Kubelet during admission if Node has no more room.\n   * _Case 2_: Kubelet finds that the new desired Resources does not fit.\n     - If Kubelet determines there isn't enough room, it simply retries the Pod\n       resize at a later time.\n1. Scheduler uses cached Pod's Spec.Containers[i].ResourcesAllocated to compute\n   resources available on the Node while a Pod resize may be in progress.\n   * If a new Pod is assigned to that Node in parallel, it can temporarily\n     result in actual sum of Pod resources for the Node exceeding Node's\n     allocable resources. This is resolved when Kubelet rejects that new Pod\n     during admission due to lack of room.\n   * Once Kubelet that accepted a parallel Pod resize updates that Pod's\n     Spec.Containers[i].ResourcesAllocated, and subsequently the Scheduler\n     updates its cache, accounting will reflect updated Pod resources for\n     future computations and scheduling decisions.\n1. The initiating actor (e.g. VPA) observes the following:\n   * _Case 1_: Pod's Spec.Containers[i].ResourcesAllocated values have changed\n     and matches Spec.Containers[i].Resources.Requests, signifying that desired\n     resize has been accepted, and Pod is being resized. The resize operation\n     is complete when Pod's Status.ContainerStatuses[i].Resources and\n     Spec.Containers[i].Resources match.\n   * _Case 2_: Pod's Spec.Containers[i].ResourcesAllocated remains unchanged,\n     and continues to differ from desired Spec.Containers[i].Resources.Requests.\n     After a certain (user defined) timeout, initiating actor may take alternate\n     action. For example, based on Retry policy, initiating actor may:\n     - Evict the Pod to trigger a replacement Pod with new desired resources,\n     - Do nothing and let Kubelet back off and later retry the in-place resize.\n\n#### Container resource limit update ordering\n\nWhen in-place resize is requested for multiple Containers in a Pod, Kubelet\nupdates resource limit for the Pod and its Containers in the following manner:\n  1. If resource resizing results in net-increase of a resource type (CPU or\n     Memory), Kubelet first updates Pod-level cgroup limit for the resource\n     type, and then updates the Container resource limit.\n  1. If resource resizing results in net-decrease of a resource type, Kubelet\n     first updates the Container resource limit, and then updates Pod-level\n     cgroup limit.\n  1. If resource update results in no net change of a resource type, only the\n     Container resource limits are updated.\n\nIn all the above cases, Kubelet applies Container resource limit decreases\nbefore applying limit increases.\n\n#### Notes\n\n* If CPU Manager policy for a Node is set to 'static', then only integral\n  values of CPU resize are allowed. If non-integral CPU resize is requested\n  for a Node with 'static' CPU Manager policy, that resize is rejected, and\n  an error message is logged to the event stream.\n* To avoid races and possible gamification, all components will use Pod's\n  Spec.Containers[i].ResourcesAllocated when computing resources used by Pods.\n* If additional resize requests arrive when a Pod is being resized, those\n  requests are handled after completion of the resize that is in progress. And\n  resize is driven towards the latest desired state.\n* Lowering memory limits may not always take effect quickly if the application\n  is holding on to pages. Kubelet will use a control loop to set the memory\n  limits near usage in order to force a reclaim, and update the Pod's\n  Status.ContainerStatuses[i].Resources only when limit is at desired value.\n* Impact of Pod Overhead: Kubelet adds Pod Overhead to the resize request to\n  determine if in-place resize is possible.\n* Impact of memory-backed emptyDir volumes: If memory-backed emptyDir is in\n  use, Kubelet will clear out any files in emptyDir upon Container restart.\n* At this time, Vertical Pod Autoscaler should not be used with Horizontal Pod\n  Autoscaler on CPU, memory. This enhancement does not change that limitation.\n\n### Affected Components\n\nPod v1 core API:\n* extended model,\n* new subresource,\n* added validation.\n\nAdmission Controllers: LimitRanger, ResourceQuota need to support Pod Updates:\n* for ResourceQuota, podEvaluator.Handler implementation is modified to allow\n  Pod updates, and verify that sum of Pod.Spec.Containers[i].Resources for all\n  Pods in the Namespace don't exceed quota,\n* for LimitRanger we check that a resize request does not violate the min and\n  max limits specified in LimitRange for the Pod's namespace.\n\nKubelet:\n* set Pod's Status.ContainerStatuses[i].Resources for Containers upon placing\n  a new Pod on the Node,\n* update Pod's Spec.Containers[i].ResourcesAllocated upon resize,\n* change UpdateContainerResources CRI API to work for both Linux \u0026 Windows.\n\nScheduler:\n* compute resource allocations using Pod.Spec.Containers[i].ResourcesAllocated.\n\nControllers:\n* propagate Template resources update to running Pod instances.\n\nOther components:\n* check how the change of meaning of resource requests influence other\n  Kubernetes components.\n\n### Future Enhancements\n\n1. Kubelet (or Scheduler) evicts lower priority Pods from Node to make room for\n   resize. Pre-emption by Kubelet may be simpler and offer lower latencies.\n1. Allow ResizePolicy to be set on Pod level, acting as default if (some of)\n   the Containers do not have it set on their own.\n1. Extend ResizePolicy to separately control resource increase and decrease\n   (e.g. a Container can be given more memory in-place but decreasing memory\n   requires Container restart).\n1. Extend Node Information API to report the CPU Manager policy for the Node,\n   and enable validation of integral CPU resize for nodes with 'static' CPU\n   Manager policy.\n1. Allow resizing local ephemeral storage.\n1. Allow resource limits to be updated (VPA feature).\n\n### Risks and Mitigations\n\n1. Backward compatibility: When Pod.Spec.Containers[i].Resources becomes\n   representative of desired state, and Pod's true resource allocations are\n   tracked in Pod.Spec.Containers[i].ResourcesAllocated, applications that\n   query PodSpec and rely on Resources in PodSpec to determine resource\n   allocations will see values that may not represent actual allocations. As\n   a mitigation, this change needs to be documented and highlighted in the\n   release notes, and in top-level Kubernetes documents.\n1. Resizing memory lower: Lowering cgroup memory limits may not work as pages\n   could be in use, and approaches such as setting limit near current usage may\n   be required. This issue needs further investigation.\n\n## Graduation Criteria\n\nTODO\n\n## Implementation History\n\n- 2018-11-06 - initial KEP draft created\n- 2019-01-18 - implementation proposal extended\n- 2019-03-07 - changes to flow control, updates per review feedback\n- 2019-08-29 - updated design proposal\n"
+  },
+  {
+    "id": "f2786a84edce1cc0222a8314787d034e",
+    "title": "Configurable scale up/down velocity for HPA",
+    "authors": ["@gliush", "@arjunrn"],
+    "owningSig": "sig-autoscaling",
+    "participatingSigs": null,
+    "reviewers": ["TBD"],
+    "approvers": ["TBD"],
+    "editor": "TBD",
+    "creationDate": "2019-03-07",
+    "lastUpdated": "2019-09-16",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Configurable scale up/down velocity for HPA\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Story 1: Scale Up As Fast As Possible](#story-1-scale-up-as-fast-as-possible)\n    - [Story 2: Scale Up As Fast As Possible, Scale Down Very Gradually](#story-2-scale-up-as-fast-as-possible-scale-down-very-gradually)\n    - [Story 3: Scale Up Very Gradually, Usual Scale Down Process](#story-3-scale-up-very-gradually-usual-scale-down-process)\n    - [Story 4: Scale Up As Usual, Do Not Scale Down](#story-4-scale-up-as-usual-do-not-scale-down)\n    - [Story 5: Stabilization before scaling down](#story-5-stabilization-before-scaling-down)\n    - [Story 6: Avoid false positive signals for scaling up](#story-6-avoid-false-positive-signals-for-scaling-up)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n    - [Algorithm Pseudocode](#algorithm-pseudocode)\n    - [Introducing \u003ccode\u003estabilizationWindowSeconds\u003c/code\u003e Option](#introducing--option)\n    - [Default Values](#default-values)\n    - [Stabilization Window](#stabilization-window)\n    - [API Changes](#api-changes)\n    - [HPA Controller State Changes](#hpa-controller-state-changes)\n    - [Command Line Options Changes](#command-line-options-changes)\n- [Graduation Criteria](#graduation-criteria)\n\u003c!-- /toc --\u003e\n\n## Summary\n\n[Horizontal Pod Autoscaler][] (HPA) automatically scales the number of pods in any resource which supports the `scale` subresource based on observed CPU utilization\n(or, with custom metrics support, on some other application-provided metrics). This proposal adds scale velocity configuration parameters to the HPA to control the\nrate of scaling in both directions.\n\n[Horizontal Pod Autoscaler]: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/\n\n## Motivation\n\nDifferent applications may have different business values, different logic and may require different scaling behaviors.\nI can name at least three types of applications:\n\n- Applications that handle business-critical web traffic. They should scale up as fast as possible (false positive signals to scale up are ok), and scale down very slowly (waiting for another traffic spike).\n- Applications that process critical data. They should scale up as fast as possible (to reduce the data processing time), and scale down as soon as possible (to reduce the costs). False positives signals to scale up/down are ok.\n- Applications that process regular data/web traffic. These are not very critical and may scale up and down in a usual way to minimize jitter.\n\nAt the moment, there’s only one cluster-level configuration parameter that influence how fast the cluster is scaled down:\n\n- [--horizontal-pod-autoscaler-downscale-stabilization-window][]   (default to 5 min)\n\nAnd a couple of hard-coded constants that specify how fast the target can scale up:\n\n- [scaleUpLimitFactor][] = 2.0\n- [scaleUpLimitMinimum][] = 4.0\n\nAs a result, users cannot influence scale velocity, and that is a problem for many applications. There are several open issues in the tracker about this issue:\n\n- [#39090][]\n- [#65097][]\n- [#69428][]\n\n[--horizontal-pod-autoscaler-downscale-stabilization-window]: https://v1-14.docs.kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#algorithm-details\n[scaleUpLimitFactor]: https://github.com/kubernetes/kubernetes/blob/release-1.14/pkg/controller/podautoscaler/horizontal.go#L55\n[scaleUpLimitMinimum]: https://github.com/kubernetes/kubernetes/blob/release-1.14/pkg/controller/podautoscaler/horizontal.go#L56\n[#39090]: https://github.com/kubernetes/kubernetes/issues/39090\n[#65097]: https://github.com/kubernetes/kubernetes/issues/65097\n[#69428]: https://github.com/kubernetes/kubernetes/issues/69428\n\n### Goals\n\n- Allow the user to be able to manage the scale velocity\n\n### Non-Goals\n\n- Persist the scaling events so that the HPA behavior is consistent even when the controller is restarted.\n- Add `tolerance` parameter to the new `behavior` section for both `scaleUp` and `scaleDown`\n\n## Proposal\n\nWe need to introduce an algorithm-agnostic HPA object configuration that will allow configuration of individual HPAs.\nTo customize the scaling behavior we should add a `behavior` object with the following fields:\n\n- `scaleUp` specifies the rules which are used to control scaling behavior while scaling up.\n  - `stabilizationWindowSeconds` - this value indicates the amount of time the HPA controller should consider\n      previous recommendations to prevent flapping of the number of replicas.\n  - `selectPolicy` can be `min` or `max` and specifies which value from the policies should be selected. The `max` value is used by default.\n  - `policies` a list of policies which regulate the amount of scaling. Each item has the following fields\n    - `type` can have the value `pods` or `percent` which indicates the allowed changed in terms of absolute number of pods or percentage of current replicas.\n    - `periodSeconds` the amount of time in seconds for which the rule should hold true.\n    - `value` the value for the policy\n- `scaleDown` similar to the `scaleUp` but specifies the rules for scaling down.\n\nA user will specify the parameters for the HPA, thus controlling the HPA logic.\n\nThe `selectPolicy` field indicates which policy should be applied. By default the `max` policy is chosen or in other words while scaling up the highest\npossible number of replicas is used and while scaling down the lowest possible number of replicas is chosen. \n\nIf the user does not specify `policies` for either `scaleUp` or `scaleDown` then default value for that policy is used \n(see the [Default Values] [] section below). Setting the `value` to `0` for `scaleUp` or `scaleDown` disables scaling in that direction.\n\n[Default Values]: #default-values\n[Stabilization Window]: #stabilization-window\n\n### User Stories\n\n#### Story 1: Scale Up As Fast As Possible\n\nThis mode is essential when you want to respond to a traffic increase quickly.\n\nCreate an HPA with the following configuration:\n\n```yaml\nbehavior:\n  scaleUp:\n    policies:\n    - type: percent\n      value: 900%\n```\n\nThe `900%` implies that 9 times the current number of pods can be added, effectively making the number\nof replicas 10 times the current size. All other parameters are not specified (default values are used)\n\nIf the application is started with 1 pod, it will scale up with the following number of pods:\n\n    1 -\u003e 10 -\u003e 100 -\u003e 1000\n\nThis way the target can reach `maxReplicas` very quickly.\n\nScale down will be done in the usual way (check stabilization window in the [Stabilization Window][] section below and the [Algorithm details][] in the official HPA documentation)\n\n[Stabilization Window]: #stabilization-window\n[Algorithm details]: https://v1-14.docs.kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#algorithm-details\n\n#### Story 2: Scale Up As Fast As Possible, Scale Down Very Gradually\n\nThis mode is essential when you do not want to risk scaling down very quickly.\n\nCreate an HPA with the following behavior:\n\n```yaml\nbehavior:\n  scaleUp:\n    policies:\n    - type: percent\n      value: 900%\n  scaleDown:\n    policies:\n    - type: pods\n      value: 1\n      periodSeconds: 600 # (i.e., scale down one pod every 10 min)\n```\n\nThis `behavior` has the same scale-up pattern as the previous example. However the `behavior` for scaling down is also specified.\nThe `scaleUp` behavior will be fast as explained in the previous example. However the target will scale down by only one pod every 10 minutes.\n\n    1000 -\u003e 1000 -\u003e 1000 -\u003e … (7 more min) -\u003e 999\n\n#### Story 3: Scale Up Very Gradually, Usual Scale Down Process\n\nThis mode is essential when you want to increase capacity, but you want it to be very pessimistic.\nCreate an HPA with the following behavior:\n\n```yaml\nbehavior:\n  scaleUp:\n    policies:\n    - type: pods\n      value: 1\n```\n\nIf the application is started with 1 pod, it will scale up very gradually:\n\n    1 -\u003e 2 -\u003e 3 -\u003e 4\n\nScale down will be done a usual way (check stabilization window in [Algorithm details][])\n\n[Algorithm details]: https://v1-14.docs.kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#algorithm-details\n\n#### Story 4: Scale Up As Usual, Do Not Scale Down\n\nThis mode is essential when scale down should not happen or should be controlled by a separate process.\n\nCreate an HPA with the following constraints:\n\n```yaml\nbehavior:\n  scaleDown:\n    policies:\n    - type: pods\n      value: 0\n```\n\nThe cluster will scale up as usual (default values), but will never scale down.\n\n#### Story 5: Stabilization before scaling down\n\nThis mode is used when the user expects a lot of flapping or does not want to scale down pods too early expecting some late load spikes.\n\nCreate an HPA with the following behavior:\n\n```yaml\nbehavior:\n  scaleDown:\n    stabilizationWindowSeconds: 600\n    policies:\n    - type: pods\n      value: 5\n```\n\ni.e., the algorithm will:\n\n- gather recommendations for 600 seconds _(default: 300 seconds)_\n- pick the largest one\n- scale down no more than 5 pods per minute\n\nExample for `CurReplicas = 10` and HPA controller cycle once per a minute:\n\n- First 9 minutes the algorithm will do nothing except gathering recommendations.\n  Let's imagine that we have the following recommendations\n\n    recommendations = [10, 9, 8, 9, 9, 8, 9, 8, 9]\n\n- On the 10th minute, we'll add one more recommendation (let it me `8`):\n\n    recommendations = [10, 9, 8, 9, 9, 8, 9, 8, 9, 8]\n\n  Now the algorithm picks the largest one `10`. Hence it will not change number of replicas\n\n- On the 11th minute, we'll add one more recommendation (let it be `7`) and removes the first one to keep the same amount of recommendations:\n\n    recommendations = [9, 8, 9, 9, 8, 9, 8, 9, 8, 7]\n\n  The algorithm picks the largest value `9` and changes the number of replicas `10 -\u003e 9`\n\n#### Story 6: Avoid false positive signals for scaling up\n\nThis mode is useful in Data Processing pipelines when the number of replicas depends on the number of events in the queue.\nThe users want to scale up quickly if they have a high number of events in the queue. \nHowever, they do not want to react to false positive signals, i.e. to short spikes of events.\n\nCreate an HPA with the following behavior:\n\n```yaml\nbehavior:\n  scaleUp:\n    stabilizationWindowSeconds: 300\n    policies:\n    - type: pods\n      value: 20\n```\n\ni.e., the algorithm will:\n\n- gather recommendations for 300 seconds _(default: 0 seconds)_\n- pick the smallest one\n- scale up no more than 20 pods per minute\n\nExample for `CurReplicas = 2` and HPA controller cycle once per a minute:\n\n- First 5 minutes the algorithm will do nothing except gathering recommendations.\n  Let's imagine that we have the following recommendations\n\n    recommendations = [2, 3, 19, 10, 3]\n\n- On the 6th minute, we'll add one more recommendation (let it me `4`):\n\n    recommendations = [2, 3, 19, 10, 3, 4]\n\n  Now the algorithm picks the smallest one `2`. Hence it will not change number of replicas\n\n- On the 7th minute, we'll add one more recommendation (let it be `7`) and removes the first one to keep the same amount of recommendations:\n\n    recommendations = [7, 3, 19, 10, 3, 4]\n\n  The algorithm picks the smallest value `3` and changes the number of replicas `2 -\u003e 3`\n\n### Implementation Details/Notes/Constraints\n\nTo minimize the impact of new changes on existing code the HPA controller will be modified in a such\na way that the scaling calculations will have separate code paths for existing HPA and HPAs with\nthe `behavior` field set. The new code path will be as shown below.\n\n#### Algorithm Pseudocode\n\nThe algorithm to find the number of pods will look like this:\n\n```golang\n  for { // infinite cycle inside the HPA controller\n    desiredReplicas = AnyAlgorithmInHPAController(...)\n    if desiredReplicas \u003e curReplicas {\n      replicas = []int{}\n      for _, policy := range behavior.ScaleUp.Policies {\n        if policy.type == \"pods\" {\n          replicas = append(replicas, CurReplicas + policy.Value)\n        } else if policy.type == \"percent\" {\n          replicas = append(CurReplicas * (1 + policy.Value/100))\n        }\n      }\n      if behavior.ScaleUp.selectPolicy == \"max\" {\n        scaleUpLimit = max(replicas)\n      } else {\n        scaleUpLimit = min(replicas)\n      }\n      limitedReplicas = min(max, desiredReplicas)\n    }\n    if desiredReplicas \u003c curReplicas {\n      for _, policy := range behaviro.scaleDown.Policies {\n        replicas = []int{}\n        if policy.type == \"pods\" {\n          replicas = append(replicas, CurReplicas - policy.Value)\n        } else if policy.type == \"percent\" {\n          replicas = append(replicas, CurReplicas * (1 - policy.Value /100))\n        }\n        if behavior.ScaleDown.SelectPolicy == \"max\" {\n          scaleDownLimit = min(replicas)\n        } else {\n          scaleDownLimit = max(replicas)\n        }\n        limitedReplicas = max(min, desiredReplicas)\n      }\n    }\n    storeRecommend(limitedReplicas, scaleRecommendations)\n    nextReplicas := applyRecommendationIfNeeded(scaleRecommendations)\n    setReplicas(nextReplicas)\n    sleep(ControllerSleepTime)\n  }\n\n```\n\nIf no scaling policy is specified then the default policy is chosen(see the [Default Values][] section).\n\n[Default Values]: #default-values\n\n#### Introducing `stabilizationWindowSeconds` Option\n\nEffectively the `stabilizationWindowSeconds` option is a full copy of the current [Stabilization Window][] algorithm extended to cover scale up:\n\n- While scaling down, we should pick the safest (largest) \"desiredReplicas\" number during last `stabilizationWindowSeconds`.\n- While scaling up, we should pick the safest (smallest) \"desiredReplicas\" number during last `stabilizationWindowSeconds`.\n\nCheck the [Algorithm Pseudocode][] section if you need more details.\n\nIf the window is `0`, it means that no delay should be used. And we should instantly change the number of replicas.\n\nIf no value is specified, the default value is used, see the [Default Values][] section.\n\nThe __“Stabilization Window\"__ as a result becomes an alias for the `behavior.scaleDown.stabilizationWindowSeconds`.\n\n[Stabilization Window]: #stabilization-window\n[Algorithm Pseudocode]: #algorithm-pseudocode\n[Default Values]: #default-values\n\n#### Default Values\n\nFor smooth transition it makes sense to set the following default values:\n\n- `behavior.scaleDown.stabilizationWindowSeconds = 300`, wait 5 min for the largest recommendation and then scale down to that value.\n- `behavior.scaleUp.stabilizationWindowSeconds = 0`, do not gather recommendations, instantly scale up to the calculated number of replicas\n- `behavior.scaleUp.policies` has the following policies\n   - Percentage policy\n      - `policy = percent`\n      - `periodSeconds = 60`, one minute period for scaleUp\n      - `value = 100` which means the number of replicas can be doubled every minute.\n   - Pod policy\n      - `policy = pods`\n      - `periodSeconds = 60`, one minute period for scaleUp\n      - `value = 4` which means the 4 replicas can be added every minute.\n- `behavior.scaleDown.policies` has the following policies\n  - Percentage Policy\n    - `policy = percent`\n    - `periodSeconds = 60` one minute period for scaleDown\n    - `value = 100`  which means all the replicas can be scaled down in one minute.\n\nPlease note that:\n\n`behavior.scaleDown.stabilizationWindowSeconds` value is picked in the following order:\n\n- from the HPA configuration if specified\n- from the command-line options for the controller. Check the [Command Line Option Changes][] section.\n- from the hardcoded default value `300`.\n\nThe `scaleDown` behavior has a single `percent` policy with a value of `100` because\nthe current scale down behavior is only limited by [Stabilization Window][] which means after\nthe stabilization window has passed the target can be scaled down to the minimum specified replicas.\nIn order to replicate the default behavior we set `behavior.scaleDown.stabilizationWindowSeconds` to 300\n(the default value for Stabilization window), and let it determine the number of pods.\n\n[Stabilization Window]: #stabilization-window\n\n#### Stabilization Window\n\nCurrently the stabilization window ([PR][], [RFC][], [Algorithm Details][]) is used to gather __scale-down-recommendations__\nduring a fixed interval (default is 5min), and a new number of replicas is set to the maximum of all recommendations\nin that interval. This is done to prevent a constant fluctuation in the number of pods if the traffic/load fluctuates\nover a short period.\n\nIt may be specified by command line option `--horizontal-pod-autoscaler-downscale-stabilization-window`.\n\n[PR]: https://github.com/kubernetes/kubernetes/pull/68122\n[RFC]: https://docs.google.com/document/d/1IdG3sqgCEaRV3urPLA29IDudCufD89RYCohfBPNeWIM/edit#heading=h.3tdw2jxiu42f\n[Algorithm Details]: https://v1-14.docs.kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#algorithm-details\n\n#### API Changes\n\nThe following API changes are needed:\n\n- `HorizontalPodAutoscalerBehavior` should be added as a field to the HPA spec. It will contain fields\n  which describe the scaling behavior for both scale up and scale down. In the future other aspects of\n  the scaling behavior could be customized by adding fields here.\n- `HPAScaleConstraint` specifies the maximum change in the number of pods while scaling up or down\n  during autoscaling.\n\nThe resulting data structures will look like this:\n\n```golang\ntype HPAScalingPolicyType string\n\nconst (\n  PercentPolicy HPAScalingPolicyType = \"percent\"\n  PodsPolicy    HPAScalingPolicyType = \"pods\"\n)\n\ntype HPAScalingPolicy struct {\n  Type          HPAScalingPolicyType\n  Value         int32\n  PeriodSeconds int32\n}\n\ntype HPAScalingRules struct {\n  StabilizationWindowSeconds *int32\n  Policies     []HpaScalingPolicy\n  SelectPolicy *string\n}\n\ntype HorizontalPodAutoscalerBehavior struct {\n    ScaleUp   *HPAScalingRules\n    ScaleDown *HPAScalingRules\n}\n\ntype HorizontalPodAutoscalerSpec struct {\n    ScaleTargetRef CrossVersionObjectReference\n    MinReplicas    *int32\n    MaxReplicas    int32\n    Metrics        []MetricSpec\n    Behavior      *HorizontalPodAutoscalerBehavior\n}\n```\n\n#### HPA Controller State Changes\n\nTo store the history of scaling events, the HPA controller needs an additional field __(similar to the list of “recommendations”)__\n\n```golang\nscaleEvents map[string][]timestampedScaleEvent\n```\n\nwhere `timestampedScaleEvent` is\n\n```golang\ntype timestampedScaleEvent struct {\n    replicaChange int32\n    timestamp     time.Time\n}\n```\n\nIt will store last scale events and will be used to make decisions about next scale actions.\n\nSay, if 30 seconds ago the number of replicas was increased by one, and we forbid to scale up for more than 1 pod per minute,\nthen during the next 30 seconds, the HPA controller will not scale up the target again.\n\nIf the controller is restarted, the state information is lost so the behavior is not guaranteed anymore and the\ncontroller may scale a target instantly after the restart.\n\nThough, I don’t think this is a problem, as:\n\n- It should not happen often, or you have some cluster problem\n- It requires quite a lot of time to start an HPA pod, for HPA pod to become live and ready, to get and process metrics.\n- If you have a large discrepancy between what is a desired number of replicas according to metrics and what is your current number of replicas and you DON’T want to scale - probably, you shouldn’t want to use the HPA. As the HPA goal is the opposite.\n- The stabilization algorithm already stores recommendations in memory and this has not yet been reported as an issue\n  so far.\n\nAs the added parameters have default values, we don’t need to update the API version, and may stay on the same `pkg/apis/autoscaling/v2beta2`.\n\n#### Command Line Options Changes\n\nIt should be noted that the current [--horizontal-pod-autoscaler-downscale-stabilization-window][] option defines the default value for the\n`behavior.stabilizationWindowSeconds`. As it becomes part of the HPA specification, the option is not needed anymore.\nSo we should make it obsolete but we should keep the existing flag till user have a chance to migrate.\n\nCheck the [Default Values][] section for more information about how to determine the delay (priorities of options).\n\n[--horizontal-pod-autoscaler-downscale-stabilization-window]: https://v1-14.docs.kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#algorithm-details\n[DefaultValues]: #default-values\n\n## Graduation Criteria\n\nAll the new configuration will be added to the `autoscaling/v2beta2` API which has not yet graduated to GA. So these changes do not need a separate\nGraduation Criteria and will be part of the existing beta API.\n"
+  },
+  {
+    "id": "b64644b20c60f0129fe1527bbb9624e7",
+    "title": "Kustomize",
+    "authors": ["@pwittrock", "@monopole"],
+    "owningSig": "sig-cli",
+    "participatingSigs": ["sig-cli"],
+    "reviewers": ["@droot"],
+    "approvers": ["@soltysh"],
+    "editor": "@droot",
+    "creationDate": "2018-05-05",
+    "lastUpdated": "2019-01-09",
+    "status": "implemented",
+    "seeAlso": ["n/a"],
+    "replaces": ["kinflate"],
+    "supersededBy": ["kustomize-subbcommand-integration.md"],
+    "markdown": "\n# Kustomize\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n    - [Long standing issues](#long-standing-issues)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n    - [Exposing every imperative kubectl command in a declarative fashion](#exposing-every-imperative-kubectl-command-in-a-declarative-fashion)\n    - [Providing a simpler facade on top of the Kubernetes APIs](#providing-a-simpler-facade-on-top-of-the-kubernetes-apis)\n- [Proposal](#proposal)\n  - [Capabilities](#capabilities)\n    - [\u003cem\u003ekustomization.yaml\u003c/em\u003e will allow users to reference config files](#kustomizationyaml-will-allow-users-to-reference-config-files)\n    - [\u003cem\u003ekustomization.yaml\u003c/em\u003e will allow users to generate configs from files](#kustomizationyaml-will-allow-users-to-generate-configs-from-files)\n    - [\u003cem\u003ekustomization.yaml\u003c/em\u003e will allow users to apply transformations to configs](#kustomizationyaml-will-allow-users-to-apply-transformations-to-configs)\n  - [UX](#ux)\n    - [Edit](#edit)\n    - [Diff](#diff)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n  - [Risks and Mitigations](#risks-and-mitigations)\n  - [Risks of Not Having a Solution](#risks-of-not-having-a-solution)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Drawbacks](#drawbacks)\n- [Alternatives](#alternatives)\n- [FAQs](#faqs)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nDeclarative specification of Kubernetes objects is the recommended way to manage Kubernetes\nproduction workloads, however gaps in the kubectl tooling force users to write their own scripting and\ntooling to augment the declarative tools with preprocessing transformations.\nWhile most of these transformations already exist as imperative kubectl commands, they are not natively accessible\nfrom a declarative workflow.\n\nThis KEP describes how `kustomize` addresses this problem by providing a declarative format for users to access\nthe imperative kubectl commands they are already familiar natively from declarative workflows.\n\n## Motivation\n\nThe kubectl command provides a cli for:\n\n- accessing the Kubernetes apis through json or yaml configuration\n- porcelain commands for generating and transforming configuration off of command line flags.\n\nExamples:\n\n- Generate a configmap or secret from a text or binary file\n  - `kubectl create configmap`, `kubectl create secret`\n  - Users can manage their configmaps and secrets text and binary files\n\n- Create or update fields that cut across other fields and objects\n  - `kubectl label`, `kubectl annotate`\n  - Users can add and update labels for all objects composing an application\n\n- Transform an existing declarative configuration without forking it\n  - `kubectl patch`\n  - Users may generate multiple variations of the same workload\n\n- Transform live resources arbitrarily without auditing\n  - `kubectl edit`\n\nTo create a Secret from a binary file, users must first base64 encode the binary file and then create a Secret yaml\nconfig from the resulting data.  Because the source of truth is actually the binary file, not the config,\nusers must write scripting and tooling to keep the 2 sources consistent.\n\nInstead, users should be able to access the simple, but necessary, functionality available in the imperative\nkubectl commands from their declarative workflow.\n\n#### Long standing issues\n\nKustomize addresses a number of long standing issues in kubectl.\n\n- Declarative enumeration of multiple files [kubernetes/kubernetes#24649](https://github.com/kubernetes/kubernetes/issues/24649)\n- Declarative configmap and secret creation: [kubernetes/kubernetes#24744](https://github.com/kubernetes/kubernetes/issues/24744), [kubernetes/kubernetes#30337](https://github.com/kubernetes/kubernetes/issues/30337)\n- Configmap rollouts: [kubernetes/kubernetes#22368](https://github.com/kubernetes/kubernetes/issues/22368)\n  - [Example in kustomize](https://github.com/kubernetes-sigs/kustomize/tree/master/examples/helloWorld#how-this-works-with-kustomize)\n- Name/label scoping and safer pruning: [kubernetes/kubernetes#1698](https://github.com/kubernetes/kubernetes/issues/1698)\n  - [Example in kustomize](https://github.com/kubernetes-sigs/kustomize/blob/master/examples/breakfast.md#demo-configure-breakfast)\n- Template-free add-on customization: [kubernetes/kubernetes#23233](https://github.com/kubernetes/kubernetes/issues/23233)\n  - [Example in kustomize](https://github.com/kubernetes-sigs/kustomize/tree/master/examples/helloWorld#staging-kustomization)\n\n### Goals\n\n- Declarative support for defining ConfigMaps and Secrets generated from binary and text files\n- Declarative support for adding or updating cross-cutting fields\n  - labels \u0026 selectors\n  - annotations\n  - names (as transformation of the original name)\n- Declarative support for applying patches to transform arbitrary fields\n  - use strategic-merge-patch format\n- Ease of integration with CICD systems that maintain configuration in a version control repository\n  as a single source of truth, and take action (build, test, deploy, etc.) when that truth changes (gitops).\n\n### Non-Goals\n\n#### Exposing every imperative kubectl command in a declarative fashion\n\nThe scope of kustomize is limited only to functionality gaps that would otherwise prevent users from\ndefining their workloads in a purely declarative manner (e.g. without writing scripts to perform pre-processing\nor linting).  Commands such as `kubectl run`, `kubectl create deployment` and `kubectl edit` are unnecessary\nin a declarative workflow because a Deployment can easily be managed as declarative config.\n\n#### Providing a simpler facade on top of the Kubernetes APIs\n\nThe community has developed a number of facades in front of the Kubernetes APIs using\ntemplates or DSLs.  Attempting to provide an alternative interface to the Kubernetes API is\na non-goal.  Instead the focus is on:\n\n- Facilitating simple cross-cutting transformations on the raw config that would otherwise require other tooling such\n  as *sed*\n- Generating configuration when the source of truth resides elsewhere\n- Patching existing configuration with transformations\n\n## Proposal\n\n### Capabilities\n\n**Note:** This proposal has already been implemented in `github.com/kubernetes/kubectl`.\n\nDefine a new meta config format called *kustomization.yaml*.\n\n#### *kustomization.yaml* will allow users to reference config files\n\n- Path to config yaml file (similar to `kubectl apply -f \u003cfile\u003e`)\n- Urls to config yaml file (similar to `kubectl apply -f \u003curl\u003e`)\n- Path to *kustomization.yaml* file (takes the output of running kustomize)\n\n#### *kustomization.yaml* will allow users to generate configs from files\n\n- ConfigMap (`kubectl create configmap`)\n- Secret (`kubectl create secret`)\n\n#### *kustomization.yaml* will allow users to apply transformations to configs\n\n- Label (`kubectl label`)\n- Annotate (`kubectl annotate`)\n- Strategic-Merge-Patch (`kubectl patch`)\n- Name-Prefix\n\n### UX\n\nKustomize will also contain subcommands to facilitate authoring *kustomization.yaml*.\n\n#### Edit\n\nThe edit subcommands will allow users to modify the *kustomization.yaml* through cli commands containing\nhelpful messaging and documentation.\n\n- Add ConfigMap - like `kubectl create configmap` but declarative in *kustomization.yaml*\n- Add Secret - like `kubectl create secret` but declarative in *kustomization.yaml*\n- Add Resource - adds a file reference to *kustomization.yaml*\n- Set NamePrefix - adds NamePrefix declaration to *kustomization.yaml*\n\n#### Diff\n\nThe diff subcommand will allow users to see a diff of the original and transformed configuration files\n\n- Generated config (configmap) will show the files as created\n- Transformations (name prefix) will show the files as modified\n\n### Implementation Details/Notes/Constraints [optional]\n\nKustomize has already been implemented in the `github.com/kubernetes/kubectl` repo, and should be moved to a\nseparate repo for the subproject.\n\nKustomize was initially developed as its own cli, however once it has matured, it should be published\nas a subcommand of kubectl or as a statically linked plugin.  It should also be more tightly integrated with apply.\n\n- Create the *kustomize* sig-cli subproject and update sigs.yaml\n- Move the existing kustomize code from `github.com/kubernetes/kubectl` to `github.com/kubernetes-sigs/kustomize`\n\n### Risks and Mitigations\n\n\n### Risks of Not Having a Solution\n\nBy not providing a viable option for working directly with Kubernetes APIs as json or\nyaml config, we risk the ecosystem becoming fragmented with various bespoke API facades.\nBy ensuring the raw Kubernetes API json or yaml is a usable approach for declaratively\nmanaging applications, even tools that do not use the Kubernetes API as their native format can\nbetter work with one another through transformation to a common format.\n\n## Graduation Criteria\n\n- Dogfood kustomize by either:\n  - moving one or more of our own (OSS Kubernetes) services to it.\n  - getting user feedback from one or more mid or large application deployments using kustomize.\n- Publish kustomize as a subcommand of kubectl.\n\n## Implementation History\n\nkustomize was implemented in the kubectl repo before subprojects became a first class thing in Kubernetes.\nThe code has been fully implemented, but it must be moved to a proper location.\n\n## Drawbacks\n\n\n## Alternatives\n\n1. Users write their own bespoke scripts to generate and transform the config before it is applied.\n2. Users don't work with the API directly, and use or develop DSLs for interacting with Kubernetes.\n\n## FAQs\n"
+  },
+  {
+    "id": "20e430c30bd476d39f637f7f8ecc77ee",
+    "title": "Kubectl Plugins",
+    "authors": ["@juanvallejo"],
+    "owningSig": "sig-cli",
+    "participatingSigs": ["sig-cli"],
+    "reviewers": ["@pwittrock", "@deads2k", "@liggitt", "@soltysh"],
+    "approvers": ["@pwittrock", "@soltysh"],
+    "editor": "juanvallejo",
+    "creationDate": "2018-07-24",
+    "lastUpdated": "2010-02-26",
+    "status": "implemented",
+    "seeAlso": ["n/a"],
+    "replaces": [
+      "https://github.com/kubernetes/community/blob/master/contributors/design-proposals/cli/kubectl-extension.md",
+      "https://github.com/kubernetes/community/pull/481"
+    ],
+    "supersededBy": ["n/a"],
+    "markdown": "\n# Kubectl Plugins\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Limitations of the Existing Design](#limitations-of-the-existing-design)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Scenarios](#scenarios)\n  - [Implementation Details/Design](#implementation-detailsdesign)\n    - [Naming Conventions](#naming-conventions)\n  - [Implementation Notes/Constraints](#implementation-notesconstraints)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Drawbacks](#drawbacks)\n- [Future Improvements/Considerations](#future-improvementsconsiderations)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis proposal introduces the main design for a plugin mechanism in `kubectl`.\nThe mechanism is a git-style system, that looks for executables on a user's `$PATH` whose name begins with `kubectl-`.\nThis allows plugin binaries to override existing command paths and add custom commands and subcommands to `kubectl`.\n\n## Motivation\n\nThe main motivation behind a plugin system for `kubectl` stems from being able to provide users with a way to extend\nthe functionality of `kubectl`, beyond what is offered by its core commands.\n\nBy picturing the core commands provided by `kubectl` as essential building blocks for interacting with a Kubernetes\ncluster, we can begin to think of plugins as a means of using these building blocks to provide more complex functionality.\nA new command, `kubectl set-ns`, for example, could take advantage of the rudimentary functionality already provided by\nthe `kubectl config` command, and build on top of it to provide users with a powerful, yet easy-to-use way of switching\nto a new namespace.\n\nFor example, the user experience for switching namespaces could go from:\n\n```bash\nkubectl config set-context $(kubectl config current-context) --namespace=mynewnamespace\n```\n\nto:\n\n```\nkubectl set-ns mynewnamespace\n```\n\nwhere `set-ns` would be a user-provided plugin which would call the initial `kubectl config set-context ...` command\nand set the namespace flag according to the value provided as the plugin's first parameter.\n\nThe `set-ns` command above could have multiple variations, or be expanded to support subcommands with relative ease.\nSince plugins would be distributed by their authors, independent from the core Kubernetes repository, plugins could\nrelease updates and changes at their own pace.\n\n### Limitations of the Existing Design\n\nThe existing alpha plugin system in `kubectl` presents a few limitations with its current design.\nIt forces plugin scripts and executables to exist in a pre-determined location, requires a per-plugin metadata file for\ninterpretation, and does not provide a clear way to override existing command paths or provide additional subcommands\nwithout having to override a top-level command.\n\nThe proposed git-style re-design of the plugin system allows us to implement extensibility requests from users that the\ncurrent system is unable to address.\nSee https://github.com/kubernetes/kubernetes/issues/53640 and https://github.com/kubernetes/kubernetes/issues/55708.\n\n### Goals\n\n* Avoid any kind of installation process (no additional config, users drop an executable in their `PATH`, for example,\n  and they are then able to use that plugin with `kubectl`).\n  No additional configuration is needed, only the plugin executable.\n  A plugin's filename determines the plugin's intention, such as which path in the command tree it applies to:\n  `/usr/bin/kubectl-educate-dolphins` would, for example be invoked under the command `kubectl educate dolphins --flag1 --flag2`.\n  It is up to a plugin to parse any arguments and flags given to it. A plugin decides when an argument is a\n  subcommand, as well as any limitations or constraints that its flags should have.\n* Relay all information given to `kubectl` (via command line args) to plugins as-is.\n  Plugins receive all arguments and flags provided by users and are responsible for adjusting their behavior\n  accordingly.\n* Provide a way to limit which command paths can and cannot be overridden by plugins in the command tree.\n\n### Non-Goals\n\n* The new plugin mechanism will not be a \"plugin installer\" or wizard. It will not have specific or baked-in knowledge\n  regarding a plugin's location or composition, nor will it provide a way to download or unpack plugins in a correct\n  location.\n* Plugin discovery is not a main focus of this mechanism. As such, it will not attempt to collect data about every\n  plugin that exists in an environment.\n* Plugin management is out of the scope of this design. A mechanism for updating and managing lifecycle of existing\n  plugins should be covered as a separate design (See https://github.com/kubernetes/community/pull/2340).\n* Provide a standard package of common cli utilities that is consumed by `kubectl` and plugins alike.\n  This should be done as an independent effort of this plugin mechanism.\n\n## Proposal\n\n### Scenarios\n\n* Developer wants to create and expose a plugin to `kubectl`.\n  They use a programming language of their choice and create an executable file.\n  The executable's filename consists of the command path to implement, and is prefixed with `kubectl-`.\n  The executable file is placed on the user's `PATH`.\n\n### Implementation Details/Design\n\nThe proposed design passes through all environment variables, flags, input, and output streams exactly as they are given\nto the parent `kubectl` process. This has the effect of letting plugins run without the need for any special parsing\nor case-handling in `kubectl`.\n\nIn essence, a plugin binary must be able to run as a standalone process, completely independent of `kubectl`.\n\n* When `kubectl` is executed with a subcommand _foo_ that does not exist in the command tree, it will attempt to look\nfor a filename `kubectl-foo` (`kubectl-foo.exe` on Windows) in the user's `PATH` and execute it, relaying all arguments given\nas well as all environment variables to the plugin child-process.\n\nA brief example (not an actual prototype) is provided below to clarify the core logic of the proposed design:\n\n```go\n// treat all args given by the user as pieces of a plugin binary's filename\n// and short-circuit once we find an arg that appears to be a flag.\nremainingArgs := []string{} // all \"non-flag\" arguments\n\nfor idx := range cmdArgs {\n\tif strings.HasPrefix(cmdArgs[idx], \"-\") {\n\t\tbreak\n\t}\n\tremainingArgs = append(remainingArgs, strings.Replace(cmdArgs[idx], \"-\", \"_\", -1))\n}\n\nfoundBinaryPath := \"\"\n\n// find binary in the user's PATH, starting with the longest possible filename\n// based on the given non-flag arguments by the user\nfor len(remainingArgs) \u003e 0 {\n\tpath, err := exec.LookPath(fmt.Sprintf(\"kubectl-%s\", strings.Join(remainingArgs, \"-\")))\n\tif err != nil || len(path) == 0 {\n\t\tremainingArgs = remainingArgs[:len(remainingArgs)-1]\n\t\tcontinue\n\t}\n\n\tfoundBinaryPath = path\n\tbreak\n}\n\n// if we are able to find a suitable plugin executable, perform a syscall.Exec call\n// and relay all remaining arguments (in order given), as well as environment vars.\nsyscall.Exec(foundBinaryPath, append([]string{foundBinaryPath}, cmdArgs[len(remainingArgs):]...), os.Environ())\n```\n\n#### Naming Conventions\n\nUnder this proposal, `kubectl` would identify plugins by looking for filenames beginning with the `kubectl-` prefix.\nA search for these names would occur on a user's `PATH`. Only files that are executable and begin with this prefix\nwould be identified.\n\n### Implementation Notes/Constraints\n\nThe current implementation details for the proposed design rely on using a plugin executable's name to determine what\ncommand the plugin is adding.\nFor a given command `kubectl foo --bar baz`, an executable `kubectl-foo` will be matched on a user's `PATH`,\nand the arguments `--bar baz` will be passed to it in that order.\n\nA potential limitation of this could present itself in the order of arguments provided by a user.\nA user could intend to run a plugin `kubectl-foo-bar` with the flag `--baz` with the following command\n`kubectl foo --baz bar`, but instead end up matching `kubectl-foo` with the flag `--baz` and the argument `bar` based\non the placement of the flag `--baz`.\n\nA notable constraint of this design is that it excludes any form of plugin lifecycle management, or version compatibility.\nA plugin may depend on other plugins based on the decision of a plugin author, however the proposed design does nothing\nto facilitate such dependencies. It is up to the plugin's author (or a separate / independent plugin management system) to\nprovide documentation or instructions on how to meet any dependencies required by a plugin.\n\nFurther, with the proposed design, plugins that rely on multiple \"helper\" files to properly function, should provide an\n\"entrypoint\" executable (which is placed on a user's `PATH`), with any additional files located elsewhere (e.g. ~/.kubeplugins/myplugin/helper1.py).\n\n### Risks and Mitigations\n\nUnlike the existing alpha plugin mechanism, the proposed design does not constrain commands added by plugins to exist as subcommands of the\n`kubectl plugin` design. Commands provided by plugins under the new mechanism can be invoked as first-class commands (`/usr/bin/kubectl-foo` provides the `kubectl foo` parent command).\n\nA potential risk associated with this could present in the form of a \"land-rush\" by plugin providers.\nMultiple plugin authors would be incentivized to provide their own version of plugin `foo`.\nUsers would be at the mercy of whichever variation of `kubectl-foo` is discovered in their `PATH` first when executing that command.\n\nA way to mitigate the above scenario would be to have users take advantage of the proposed plugin mechanism's design by renaming multiple variations of `kubectl-foo`\nto include the provider's name, for example: `kubectl-acme-foo`, or `kubectl-companyB-foo`.\n\nConflicts such as this one could further be mitigated by a plugin manager, which could perform conflict resolution among similarly named plugins on behalf of a user.\n\n## Graduation Criteria\n\n* Make this mechanism a part of `kubectl`'s command-lookup logic.\n\n## Implementation History\n\nThis plugin design closely follows major aspects of the plugin system design for `git`.\n\n## Drawbacks\n\nImplementing this design could potentially conflict with any ongoing work that depends on the current alpha plugin system.\n\n## Future Improvements/Considerations\n\nThe proposed design is flexible enough to accommodate future updates that could allow certain command paths to be overwritten\nor extended (with the addition of subcommands) via plugins.\n"
+  },
+  {
+    "id": "09dd1d50c3fb1f9d6b3e553b9b575834",
+    "title": "Data Driven Commands for Kubectl",
+    "authors": ["@pwittrock"],
+    "owningSig": "sig-cli",
+    "participatingSigs": null,
+    "reviewers": ["@soltysh", "@juanvallejo", "@seans3 "],
+    "approvers": ["@soltysh"],
+    "editor": "TBD",
+    "creationDate": "2018-11-13",
+    "lastUpdated": "2018-11-13",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# data driven commands\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Details](#implementation-details)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nMany Kubectl commands make requests to specific Resource endpoints.  The request bodies are populated by flags\nprovided by the user.\n\nExamples:\n\n- `create \u003cresource\u003e`\n- `set \u003cfield\u003e \u003cresource\u003e`\n- `logs`\n\nAlthough these commands are compiled into the kubectl binary, their workflow is similar to a form on a webpage and\ncould be complete driven by the server providing the client with the request (endpoint + body) and a set of flags\nto populate the request body.\n\nPublishing commands as data from the server addresses cli integration with API extensions as well as client-server\nversion skew.\n\n**Note:** No server-side changes are required for this, all Request and Response template expansion is performed on\nthe client side.\n\n## Motivation\n\nKubectl provides a number of commands to simplify working with Kubernetes by making requests to\nResources and SubResources.  These requests are mostly static, with fields filled in by user\nsupplied flags.  Today the commands are compiled into the client, which as the following challenges:\n\n- Extension APIs cannot be compiled into the client\n- Version-Skewed clients (old client) may be missing commands for new APIs or send outdated requests\n- Version-Skewed clients (new client) may have commands for APIs that are not present in the server or expose\n  fields not present in older API versions\n\n### Goals\n\nAllow client commands that make a single request to a specific resource and output the result to be data driven\nfrom the server.\n\n- Address cli support for extension APIs\n- Address user experience for version skewed clients\n\n### Non-Goals\n\nAllow client commands that have complex client-side logic to be data driven.\n\n- Require a TTY\n- Are Agnostic to Specific Resources\n\n## Proposal\n\nDefine a format for publishing simple cli commands as data.  CLI commands would be limited to:\n\n- Sending one or more requests to Resource or SubResource Endpoints\n- Populating requests from command line flags and arguments\n- Writing output populated from the Responses\n\n**Proof of Concept:** [cnctl](https://github.com/pwittrock/kubectl/tree/cnctl/cmd/cnctl)\n\nInstructions to run PoC:\n\n- `go run ./main.go` (no commands show up)\n- `kubectl apply` the `cli_v1alpha1_clitestresource.yaml` (add the CRD with the commands)\n- `go run ./main.go` (create command shows up)\n- `go run ./main create deployment -h` (view create command help)\n- `go run ./main create deploy --image nginx --name nginx` (create a deployment)\n- `kubectl get deployments`\n\n### Implementation Details\n\n**Publishing Data:**\n\nAlpha: No apimachinery changes required\n\n- Alpha: publish extension Resource Commands as an annotation on CRDs.\n- Alpha: publish core Resource Commands as openapi extension.\n\nBeta: apimachinery changes required\n\n- Beta: publish extension Resource Commands a part of the CRD Spec.\n- Beta: publish core Resource Commands from new endpoint (like *swagger.json*)\n\n**Data Command Structure:**\n\n```go\n/*\nCopyright 2018 The Kubernetes Authors.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage v1alpha1\n\ntype OutputType string\n\nconst (\n\t// Use the outputTemplate field to format the response on the client-side\n\tOUTPUT_TEMPLATE OutputType = \"TEMPLATE\"\n\n\t// Use Server-Side Printing and output the response table in a columnar format\n\tOUTPUT_TABLE                   = \"TABLE\"\n)\n\n// ResourceCommand defines a command that is dynamically defined as an annotation on a CRD\ntype ResourceCommand struct {\n\t// Command is the cli Command\n\tCommand Command `json:\"command\"`\n\n\t// Requests are the requests the command will send to the apiserver.\n\t// +optional\n\tRequests []ResourceRequest `json:\"requests,omitempty\"`\n\n\t// OutputType is used to determine what output type to print\n\t// +optional\n    OutputType OutputType `json:\"outputTemplate,omitempty\"`\n\n\t// OutputTemplate is a go-template used by the kubectl client to format the server responses as command output\n\t// (STDOUT).\n\t//\n\t// The template may reference values specified as flags using\n\t// {{index .Flags.Strings \"flag-name\"}}, {{index .Flags.Ints \"flag-name\"}}, {{index .Flags.Bools \"flag-name\"}},\n\t// {{index .Flags.Floats \"flag-name\"}}.\n\t//\n\t// The template may also reference values from the responses that were saved using saveResponseValues\n\t// {{index .Responses.Strings \"response-value-name\"}}.\n\t//\n\t// Example:\n\t// \t\tdeployment.apps/{{index .Responses.Strings \"responsename\"}} created\n\t//\n\t// +optional\n\tOutputTemplate string `json:\"outputTemplate,omitempty\"`\n}\n\ntype ResourceOperation string\n\nconst (\n\tCREATE_RESOURCE ResourceOperation = \"CREATE\"\n\tUPDATE_RESOURCE                   = \"UPDATE\"\n\tDELETE_RESOURCE                   = \"DELETE\"\n\tGET_RESOURCE                      = \"GET\"\n\tPATCH_RESOURCE                    = \"PATCH\"\n)\n\ntype ResourceRequest struct {\n\t// Group is the API group of the request endpoint\n\t//\n\t// Example: apps\n\tGroup string `json:\"group\"`\n\n\t// Version is the API version of the request endpoint\n\t//\n\t// Example: v1\n\tVersion string `json:\"version\"`\n\n\t// Resource is the API resource of the request endpoint\n\t//\n\t// Example: deployments\n\tResource string `json:\"resource\"`\n\n\t// Operation is the type of operation to perform for the request.  One of: Create, Update, Delete, Get, Patch\n\tOperation ResourceOperation `json:\"operation\"`\n\n\t// BodyTemplate is a go-template for the request Body.  It may reference values specified as flags using\n\t// {{index .Flags.Strings \"flag-name\"}}, {{index .Flags.Ints \"flag-name\"}}, {{index .Flags.Bools \"flag-name\"}},\n\t// {{index .Flags.Floats \"flag-name\"}}\n\t//\n\t// Example:\n\t//      apiVersion: apps/v1\n\t//      kind: Deployment\n\t//      metadata:\n\t//        name: {{index .Flags.Strings \"name\"}}\n\t//        namespace: {{index .Flags.Strings \"namespace\"}}\n\t//        labels:\n\t//          app: nginx\n\t//      spec:\n\t//        replicas: {{index .Flags.Ints \"replicas\"}}\n\t//        selector:\n\t//          matchLabels:\n\t//            app: {{index .Flags.Strings \"name\"}}\n\t//        template:\n\t//          metadata:\n\t//            labels:\n\t//              app: {{index .Flags.Strings \"name\"}}\n\t//          spec:\n\t//            containers:\n\t//            - name: {{index .Flags.Strings \"name\"}}\n\t//              image: {{index .Flags.Strings \"image\"}}\n\t//\n\t// +optional\n\tBodyTemplate string `json:\"bodyTemplate,omitempty\"`\n\n\t// SaveResponseValues are values read from the response and saved in {{index .Responses.Strings \"flag-name\"}}.\n\t// They may be used in the ResourceCommand.Output go-template.\n\t//\n\t// Example:\n\t//\t\t- name: responsename\n\t//        jsonPath: \"{.metadata.name}\"\n\t//\n\t// +optional\n\tSaveResponseValues []ResponseValue `json:\"saveResponseValues,omitempty\"`\n}\n\n// Flag defines a cli flag that should be registered and available in request / output templates.\n//\n// Flag is used only by the client to expand Request and Response templates with user defined values provided\n// as command line flags.\ntype Flag struct {\n\tType FlagType `json:\"type\"`\n\n\tName string `json:\"name\"`\n\n\tDescription string `json:\"description\"`\n\n\t// +optional\n\tStringValue string `json:\"stringValue,omitempty\"`\n\n\t// +optional\n\tStringSliceValue []string `json:\"stringSliceValue,omitempty\"`\n\n\t// +optional\n\tBoolValue bool `json:\"boolValue,omitempty\"`\n\n\t// +optional\n\tIntValue int32 `json:\"intValue,omitempty\"`\n\n\t// +optional\n\tFloatValue float64 `json:\"floatValue,omitempty\"`\n}\n\n// ResponseValue defines a value that should be parsed from a response and available in output templates\ntype ResponseValue struct {\n\tName     string `json:\"name\"`\n\tJsonPath string `json:\"jsonPath\"`\n}\n\ntype FlagType string\n\nconst (\n\tSTRING       FlagType = \"String\"\n\tBOOL                  = \"Bool\"\n\tFLOAT                 = \"Float\"\n\tINT                   = \"Int\"\n\tSTRING_SLICE          = \"StringSlice\"\n)\n\ntype Command struct {\n\t// Use is the one-line usage message.\n\tUse string `json:\"use\"`\n\n\t// Path is the path to the sub-command.  Omit if the command is directly under the root command.\n\t// +optional\n\tPath []string `json:\"path,omitempty\"`\n\n\t// Short is the short description shown in the 'help' output.\n\t// +optional\n\tShort string `json:\"short,omitempty\"`\n\n\t// Long is the long message shown in the 'help \u003cthis-command\u003e' output.\n\t// +optional\n\tLong string `json:\"long,omitempty\"`\n\n\t// Example is examples of how to use the command.\n\t// +optional\n\tExample string `json:\"example,omitempty\"`\n\n\t// Deprecated defines, if this command is deprecated and should print this string when used.\n\t// +optional\n\tDeprecated string `json:\"deprecated,omitempty\"`\n\n\t// Flags are the command line flags.\n\t// \n\t// Flags are used by the client to expose command line flags to users and populate the Request go-templates\n\t// with the user provided values.\n\t//\n\t// Example:\n\t// \t\t  - name: namespace\n\t//    \t\ttype: String\n\t//    \t\tstringValue: \"default\"\n\t//    \t\tdescription: \"deployment namespace\"\n\t//\n\t// +optional\n\tFlags []Flag `json:\"flags,omitempty\"`\n\n\t// SuggestFor is an array of command names for which this command will be suggested -\n\t// similar to aliases but only suggests.\n\tSuggestFor []string `json:\"suggestFor,omitempty\"`\n\n\t// Aliases is an array of aliases that can be used instead of the first word in Use.\n\tAliases []string `json:\"aliases,omitempty\"`\n\n\t// Version defines the version for this command. If this value is non-empty and the command does not\n\t// define a \"version\" flag, a \"version\" boolean flag will be added to the command and, if specified,\n\t// will print content of the \"Version\" variable.\n\t// +optional\n\tVersion string `json:\"version,omitempty\"`\n}\n\n// ResourceCommandList contains a list of Commands\ntype ResourceCommandList struct {\n\tItems []ResourceCommand `json:\"items\"`\n}\n```\n\n**Example Command:**\n\n```go\n# Set Label: \"cli.sigs.k8s.io/cli.v1alpha1.CommandList\": \"\"\n# Set Annotation: \"cli.sigs.k8s.io/cli.v1alpha1.CommandList\": '\u003cjson\u003e'\nitems:\n- command:\n    path:\n    - \"create\" # Command is a subcommand of this path\n    use: \"deployment\" # Command use\n    aliases: # Command alias'\n    - \"deploy\"\n    - \"deployments\"\n    short: Create a deployment with the specified name.\n    long: Create a deployment with the specified name.\n    example: |\n        # Create a new deployment named my-dep that runs the busybox image.\n        kubectl create deployment --name my-dep --image=busybox\n    flags:\n    - name: name\n      type: String\n      stringValue: \"\"\n      description: deployment name\n    - name: image\n      type: String\n      stringValue: \"\"\n      description: Image name to run.\n    - name: replicas\n      type: Int\n      intValue: 1\n      description: Image name to run.\n    - name: namespace\n      type: String\n      stringValue: \"default\"\n      description: deployment namespace\n  requests:\n  - group: apps\n    version: v1\n    resource: deployments\n    operation: Create\n    bodyTemplate: |\n      apiVersion: apps/v1\n      kind: Deployment\n      metadata:\n        name: {{index .Flags.Strings \"name\"}}\n        namespace: {{index .Flags.Strings \"namespace\"}}\n        labels:\n          app: nginx\n      spec:\n        replicas: {{index .Flags.Ints \"replicas\"}}\n        selector:\n          matchLabels:\n            app: {{index .Flags.Strings \"name\"}}\n        template:\n          metadata:\n            labels:\n              app: {{index .Flags.Strings \"name\"}}\n          spec:\n            containers:\n            - name: {{index .Flags.Strings \"name\"}}\n              image: {{index .Flags.Strings \"image\"}}\n    saveResponseValues:\n    - name: responsename\n      jsonPath: \"{.metadata.name}\"\n  outputTemplate: |\n    deployment.apps/{{index .Responses.Strings \"responsename\"}} created\n```\n\n### Risks and Mitigations\n\n- Command name collisions: CRD publishes command that conflicts with another command\n  - Initially require the resource name to be the command name (e.g. `create foo`, `set image foo`)\n  - Mitigation: Use the discovery service to manage preference (as it does for the K8S APIs)\n- Command makes requests on behalf of the user that may be undesirable\n  - Mitigation: Automatically output the Resource APIs that command uses as part of the command description\n  - Mitigation: Support dry-run to emit the requests made to the server without actually making them\n  - Migration: Possibly restrict the APIs commands can use (e.g. CRD published commands can only use the APIs for that\n    Resource).\n- Approach is hard to maintain, complex, etc\n  - Initially restrict to only `create` commands, get feedback\n- Doesn't work well with auto-complete\n  - TODO: Investigate if this is true and how much it matters.\n\n## Graduation Criteria\n\n- Simple commands for Core Resources have been migrated to be data driven\n- In use by high profile extension APIs - e.g. Istio\n- Published as first class item for Extension and Core Resources\n\n## Alternatives\n\n- Use plugins for these cases\n  - Still suffer from version skew\n  - Require users to download and install binaries\n  - Hard to keep in sync with set of Resources for each cluster\n- Don't support cli commands for Extension Resources\n"
+  },
+  {
+    "id": "2ed4ff623e7cb0c0f1004d5757da97ef",
+    "title": "Kustomize Generators and Transformers",
+    "authors": ["@pwittrock"],
+    "owningSig": "sig-cli",
+    "participatingSigs": null,
+    "reviewers": ["@droot", "@sethpollack"],
+    "approvers": ["@monopole"],
+    "editor": "TBD",
+    "creationDate": "2019-03-25",
+    "lastUpdated": "2019-04-30",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Kustomize Generators and Transformers\n\n\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Background](#background)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n    - [Terms](#terms)\n    - [Plugins](#plugins)\n    - [Plugins: Generate](#plugins-generate)\n    - [Plugins: Transform](#plugins-transform)\n      - [Restrictions](#restrictions)\n  - [Phases](#phases)\n  - [User Stories [optional]](#user-stories-optional)\n    - [DeclarativeEcosystem Solution Plugins](#declarativeecosystem-solution-plugins)\n    - [Bespoke Abstractions for Organizations](#bespoke-abstractions-for-organizations)\n    - [Bespoke Configuration for Organizations](#bespoke-configuration-for-organizations)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Drawbacks [optional]](#drawbacks-optional)\n- [Alternatives [optional]](#alternatives-optional)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nEnable users to extend Kustomize *Transformers* and *Generators* through `exec` plugins.  Kustomize\nprovides Resource Config to plugins through STDIN, and plugins emit Resource Config back to Kustomize\nthrough STDOUT.\n\n- Add 2 fields to `kustomization.yaml`: `generators` and `transformers`\n- Plugins to actuate new fields have the path `$XDG_CONFIG_HOME/kustomize/plugins/\u003cAPI group\u003e`\n\nMuch of the existing Kustomize functionality could be implemented through this plugin mechanism.\n\nExample Built-In Generators:\n\n- `resource`\n- `bases`\n- `secretGenerators`\n- `configMapGenerators`\n\nExample Built-In Transforms:\n\n- `commonAnnotations`\n- `commonLabels`\n- `images`\n- `namePrefix` # not supported in iteration 1 due to transformation restrictions\n- `nameSuffix` # not supported in iteration 1 due to transformation restrictions\n- `patches`\n- `namespace` # not supported in iteration 1 due to transformation restrictions\n\n\n## Motivation\n\n1. Enable organizations to develop their own authoring solutions bespoke to their requirements, and to integrate\n   those solutions with kustomize and kubectl.\n1. Enable tools and authoring solutions developed by the ecosystem to natively integrate with kustomize and kubectl\n   commands.\n1. Enable Kustomize power users to augment Kustomize with their own *Transformers* and *Generators*.\n\n### Background\n\nMany users of Kubernetes require the ability to also write Config through high-level abstractions rather\nthan only as low-level APIs, or to configure Kubernetes through in-house developed platforms.\n\nAuthoring solutions have already been developed:\n\n- In the Ecosystem\n  - Helm\n  - Ksonnet\n- Internally by Organizations\n  - [kube-gen] (AirBnB) \n\nSupport for connecting the tools developed by the Ecosystem with kubectl relies on piping commands together,\nhowever pipes are an imperative technique that require their own scripting and tooling.\n\nThis KEP proposes a declarative approach for connecting tools built in the Kubernetes ecosystem or bespoke\ninternal tools developed by organizations.\n\n[kube-gen]: https://events.linuxfoundation.org/wp-content/uploads/2017/12/Services-for-All-How-To-Empower-Engineers-with-Kubernetes-Melanie-Cebula-Airbnb.pdf\n\n### Goals\n\n- Allow Config authoring solutions developed in the ecosystem to be declaratively accessed by kubectl as\n  Generators and Transformers\n- Allow users and organizations to develop their own custom config authoring solutions and integrate\n  them into kubectl\n- Allow Kustomize power users to augment Kustomize's Built-Tranformers\n- Allow users to perform client-side mutations so they they show up in diffs and are auditable\n\n### Non-Goals\n\n- Rewrite Kustomize on top of plugins\n  - This should be possible, but isn't a problem we need to solve right now\n- Develop a market place of any sort\n  - The ecosystem solutions themselves will have market places.  This allows those ecosystem tools to\n    plug their market places into kubectl via Kustomize.\n\n## Proposal\n\nIntroduce an executable plugin that has 2 subcommands.\n\n- `$ team.example.com generate`\n- `$ team.example.com transform`\n\nIntroduce 2 new fields to `kustomization.yaml`\n\n- `generators`\n- `transformers`\n\n#### Terms\n\n- *Virtual Resource*\n  - Resource that is not a server-side Kubernetes API object, but used to configure tools\n  - Examples: kubeconfig, kustomization\n- *Generator* Kustomize directive\n  - Accepts Virtual Resources as input\n  - Emits generated non-Virtual Resources as output\n  - Examples: `configMapGenerator`, `secretGenerator`\n- *Transformer* Kustomize directive\n  - Accepts Virtual Resources\n  - Accepts **all** non-Virtual Resources as input\n  - Emits modified non-Virtual Resources as output\n  - Examples: `commonAnnotations`, `namespace`, `namePrefix`, `secretGenerator` (for updating references to the generated secret)\n- *Built-In*\n  - Either a Transformer or Generator that is part of the `kustomization.yaml` rather than\n    as a separate Virtual Resource.\n- *Plugin*\n  - Either a Transformer or Generator that is *not* part of the `kustomization.yaml` and comes from a plugin.\n\n#### Plugins\n\nGenerators and Transformers are Configured as Virtual Resources.\n\nPlugins implement Generators and Transformers.\n\n- Plugins are installed `$XDG_CONFIG_HOME/kustomize/plugins/\n- Plugin executables names match the Virtual Resource API Group the own - e.g. `team.example.com`\n- Plugin executables have 2 subcommands - `generate` and `transform`\n- `generate`\n  - Accepts 0 or more Virtual Resources whose group matches the executable name on STDIN\n  - Emits 0 or more Non-Virtual Resources\n- `transform`\n  - Accepts 0 or more Virtual Resources whose group matches the executable name on STDIN\n  - Accepts 0 or more Non-Virtual Resources from all API groups\n  - Emits 0 or more Non-Virtual Resources\n- Generators and Transformers are configured as Virtual Resources and the `generators` and `transformers` fields\n  on `kustomization.yaml`.\n- Plugins working directory is the directory of the `kustomization.yaml` with the `generator` or `transformer`.\n  - Plugins should never be able to access files outside this directory structure (e.g. only child directories)\n- Plugins inherit kustomize process environment variables\n\nPlugin guidelines:\n\n- If executed with an unrecognized subcommand, plugins should exit `127` signalling to kustomize that the\n  operation is not supported.\n- Plugins should never change state.  They should be able to be executed with `kubectl apply -k --dry-run` or\n  `kubectl diff`.\n- Plugin output should be idempotent.\n\n#### Plugins: Generate\n\nGenerators generate 0 or more Resources for some Virtual Resources.\n\nExample: Kustomize secretGenerators and configMapGenerators generate Secrets and ConfigMaps\nfrom a `kustomize.config.k8s.io/v1beta1/Kustomization` virtual Resource.\n\nGenerators have 2 components:\n\n1. Generator Config\n  - Virtual Resource\n  - Added to the `kustomization.yaml` field `generators`\n1. Generator Implementation\n  - Executable plugin\n  - Reads Virtual Resources\n  - Emits non-Virtual Resources\n\n1. Kustomize reads the `generators` Virtual Resources\n1. Kustomize maps the Virtual Resources to plugins by their *Group*\n   - Group matches the plugin name\n   - Exits non-0 if no plugins are found any generator entries\n1. For each Virtual Resource *Group*\n  1. Kustomize execs the plugin `generate` command\n  1. Kustomize writes the Virtual Resources in that Group to the process STDIN\n  1. Kustomize reads the set of generated Resources from the process STDOUT\n  1. Kustomize reads error messages from the exec process STDERR\n  1. Kustomize fails if the plugin exits non-0\n  1. Kustomize adds the emitted Resources to its set of Non-Virtual Resources (e.g. from `resources`, `bases`, etc).\n\nThe order of plugin execution is arbitrary.\n\n#### Plugins: Transform\n\nTransformers modify existing non-virtual Resources by modifying their fields.\n\nTransformers have 2 components:\n\n1. Transformer Config\n  - Virtual Resource\n  - Added to the `kustomization.yaml` field `transformers`\n  - All `generators` are implicitly invoked as transformers\n1. Transformer Implementation\n  - Executable plugin\n  - Reads Virtual Resources\n  - Reads **all** non-Virtual Resources\n  - Emits non-Virtual Resources  \n  \n1. Kustomize reads the `transformers` *and* `generators` Virtual Resources\n  - `generators` can require transformation\n1. Kustomize maps the Virtual Resources to plugins by their *Group*\n   - Group matches the plugin name\n   - Exits non-0 if no plugins are found any generator entries\n1. For each Virtual Resource *Group*\n  1. Kustomize execs the plugin `transform` command\n  1. Kustomize writes the Virtual Resources in that Group to the process STDIN\n  1. Kustomize writes **all** non-Virtual Resources to the process STDIN\n  1. Kustomize reads the set of transformed Resources from the process STDOUT\n  1. Kustomize reads error messages from the exec process STDERR\n  1. Kustomize fails if the plugin exits non-0\n  1. Kustomize replaces its current set of non-Virtual Resources with the set of emitted Resources.\n\nThe order of plugin execution is arbitrary.\n\n##### Restrictions\n\nFor the initial iteration, transformers cannot add/remove Resources, or change their names/namespaces.\nThe ability doing so would be significantly make complex ordering interactions much more likely.  E.g. transformers\nwould need to keep and propagate transformations from other transformers.\n\nWe are prioritizing a restrictive but predictable API over a powerful but complex one.\n\n### Phases\n\nFollow is the Kustomize workflow:\n\n1. Read all `generators` and `transformers`\n1. Read all `patches`\n1. Apply Virtual Resource `patches` to `generators` and `transformers`\n1. Generate non-Virtual Resource set from Built-In Generators `inputs`, `bases`, `secretGenerator`, etc\n1. Generate non-Virtual Resources from Plugin-Generators and add to non-Virtual Resource set\n1. Transform non-Virtual Resources using Plugin-Generators\n1. Transform non-Virtual Resources using Built-In Generators (including a second round of `patches`)\n1. Built-In Sorting\n\n### User Stories [optional]\n\n#### DeclarativeEcosystem Solution Plugins\n\nAlice uses GitOps for deployment of her Kubernetes Resource Config.  Alice has a Helm\nchart that she would like to include into her GitOps workflow.  Alice would like to be\nable to check the chart and values.yaml into her repository and have it deployed\njust like her other config.  Alice would also like to be able to apply cross cutting\ntransformations - such as labels, annotations, name-prefixes, etc - to the Resources generated\nby the Helm chart.\n\nAlice uses the Kustomize helm chart plugin to declaratively generate\nResource Config using Helm charts, which will then have kustomizations applied.\n\n1. Alice downloads and installs the Helm generator plugin from *github.com/kubernetes-sigs/kustomize/*\n   and installs it at `$XDG_CONFIG_HOME/kustomize/plugins/generators/helm.kustomize.io`.\n1. Alice creates the `chart.yaml` Resource Config and adds it to her `kustomization.yaml` as a `generators` entry.\n  - Having the groupVersion `helm.kustomize.io` triggers the generator under `plugins/generators/helm.kustomize.io`\n\n```yaml\ngroupVersion: helm.kustomize.io/v1beta1\nkind: Chart\ngenerate:\n    chart: ./path/to/chart/from/kustomization.yaml\n    values:\n      k1: v1\n      k2: v2\n```\n\n```yaml\napiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n\ngenerators:\n- chart.yaml\n```\n\n`$ helm.kustomize.io generate` is invoked with the `chart.yaml` contents passed on STDIN.  Kustomize\nadds the emitted Resources to its set of Resources.\n\nNote, the `transform` subcommand will also be invoked, which the plugin may choose to pass-through\nby emitting its input.\n\n#### Bespoke Abstractions for Organizations\n\nBob's organization deploys many variants of the same high-level conceptual set of Resources.  Creating the Resource Config\nfor each instance that needs to be deployed requires lots of boilerplate.  Instead Bob's organization develops\ntools for generating standardized Resource Config based off of some inputs.\n\nBob creates a new generator for his organization that allow higher level abstractions to be defined as\nnew virtual Resource types that don't exist in the cluster, but are used to generate low-level types.\n\n1. Bob builds and installs the new generator plugin `$XDG_CONFIG_HOME/kustomize/plugins/team.example.com`.\n1. Bob creates the `app.yaml` Resource Config and adds it to his `kustomization.yaml`\n\n```yaml\ngroupVersion: team.example.com/v1beta1\nkind: CommonApp\ngenerate:\n  image: foo\n  size: big\n```\n\n```yaml\napiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n\ntransformers:\n- app.yaml\n```\n\n`$ team.example.com transform` is invoked with the `app.yaml` contents passed on STDIN.  Kustomize adds the\nemitted Resource to its set of Resources.\n\n#### Bespoke Configuration for Organizations\n\nAlice's Organization requires that various fields are defaulted if unset.  SRE would like to be able to see\nthe full Resources that are being Applied and have this auditable in an scm, such as git, rather than\nhaving Webhooks provide server-side mutions that are not capture in review or scm.\n\n1. Alice builds and installs the new transformer plugin at `$XDG_CONFIG_HOME/kustomize/plugins/team.example.com`\n1. Alice creates the `transform.yaml` Resource Config and adds it to her `kustomization.yaml` as a `transformers` entry.\n\n```yaml\ngroupVersion: team.example.com\nkind: CommonApp\ntransform:\n  namePrefix: foo-\n  commonAnnotations:\n    foo: bar\n```\n\n```yaml\napiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n\ntransformers:\n- transform.yaml\n```\n\n`$ team.example.com` is invoked with both the `transform.yaml` contents and the Resources passed on STDIN.\nKustomize replaces its Resources with the emitted Resources.\n\n### Implementation Details/Notes/Constraints [optional]\n\n### Risks and Mitigations\n\n## Design Details\n\n### Test Plan\n1. Add unit tests\n2. Add integration tests\n3. Add example tests\n\n### Graduation Criteria\n\nAlpha -\u003e Beta Graduation\n1. Executing plugins for generated Resources\n2. Plugin Generators interact with Kustomize build-in transformers\n\nBeta -\u003e GA Graduation\n1. Plugin Transformers after Built-in Transformers\n2. Support ordering\n\n\n### Upgrade / Downgrade Strategy\n\nNA - Client side only\n\n### Version Skew Strategy\n\nNA - Client side only\n\n## Implementation History\n\n\n## Drawbacks [optional]\n\n- It allows users to do complex thing with Kustomize.\n- Virtual Resources may be confusing\n\n## Alternatives [optional]\n\n- Imperatively piping generator executables to kubectl apply\n- Writing scripts to invoke generator executables and piping them to kubectl apply\n- Create a new platform that is purely plug-in based, and rebase kustomize on top of this as a plugin\n- Support explicit ordering of plugin execution\n- Use separate plugins for Generators and Transformers\n- Allow arbitrary Transformation changes\n  - Increases complexity + interactions\n  - Reduces readability\n"
+  },
+  {
+    "id": "bd7d5b1dfe9434eba68ad0b1f2ac8190",
+    "title": "kubectl debug",
+    "authors": ["@verb"],
+    "owningSig": "sig-cli",
+    "participatingSigs": ["sig-cli", "sig-node"],
+    "reviewers": ["@aylei", "@soltysh"],
+    "approvers": ["@pwittrock", "@soltysh"],
+    "editor": "TBD",
+    "creationDate": "2019-08-05",
+    "lastUpdated": "2019-08-06",
+    "status": "implementable",
+    "seeAlso": [
+      "/keps/sig-node/20190212-ephemeral-containers.md",
+      "/keps/sig-release/20190316-rebase-images-to-distroless.md"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Pod Troubleshooting\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Use Cases](#use-cases)\n    - [Distroless Containers](#distroless-containers)\n    - [Kubernetes System Images](#kubernetes-system-images)\n    - [Operations and Support](#operations-and-support)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Pod Troubleshooting with Ephemeral Debug Container](#pod-troubleshooting-with-ephemeral-debug-container)\n    - [Debug Container Naming](#debug-container-naming)\n    - [Container Namespace Targeting](#container-namespace-targeting)\n    - [Interactive Troubleshooting and Automatic Attaching](#interactive-troubleshooting-and-automatic-attaching)\n    - [Proposed Ephemeral Debug Arguments](#proposed-ephemeral-debug-arguments)\n  - [Pod Troubleshooting by Copy](#pod-troubleshooting-by-copy)\n    - [Creating a Debug Container by copy](#creating-a-debug-container-by-copy)\n    - [Modify Application Image by Copy](#modify-application-image-by-copy)\n  - [Node Troubleshooting with Privileged Containers](#node-troubleshooting-with-privileged-containers)\n  - [User Stories](#user-stories)\n    - [Operations](#operations)\n    - [Debugging](#debugging)\n    - [Automation](#automation)\n    - [Technical Support](#technical-support)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e., [kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be updated for the enhancement to be released.\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nThis proposal adds a command to `kubectl` to improve the user experience of\ntroubleshooting. Current `kubectl` commands such as `exec` and `port-forward`\nallow troubleshooting at the container and network level. `kubectl debug`\nextends these capabilities to include Kubernetes abstractions such as Pod, Node\nand [Ephemeral Containers].\n\nUser journeys supported by the initial release of `kubectl debug` are:\n\n1. Create an Ephemeral Container in a running Pod to attach debugging tools\n   to a distroless container image. (See [*Pod Troubleshooting with Ephemeral\n   Containers*](#pod-troubleshooting-with-ephemeral-debug-container))\n2. Restart a pod with a modified `PodSpec`, to allow in-place troubleshooting\n   using different container images or permissions. (See [Pod Troubleshooting by\n   Copy](#pod-troubleshooting-by-copy))\n3. Start and attach to a privileged container in the host namespace. (See [*Node\n   Troubleshooting with Privileged Containers*](\n   #node-troubleshooting-with-privileged-containers))\n\n[Ephemeral Containers]: https://git.k8s.io/enhancements/keps/sig-node/20190212-ephemeral-containers.md\n\n## Motivation\n\n### Use Cases\n\n#### Distroless Containers\n\nMany developers of native Kubernetes applications wish to treat Kubernetes as an\nexecution platform for custom binaries produced by a build system. These users\ncan forgo the scripted OS install of traditional Dockerfiles and instead `COPY`\nthe output of their build system into a container image built `FROM scratch` or\na [distroless container image]. This confers several advantages:\n\n1.  **Minimal images** lower operational burden and reduce attack vectors.\n2.  **Immutable images** improve correctness and reliability.\n3.  **Smaller image size** reduces resource usage and speeds deployments.\n\nThe disadvantage of using containers built `FROM scratch` is the lack of system\nbinaries provided by a Linux distro image makes it difficult to\ntroubleshoot running containers. Kubernetes should enable one to troubleshoot\npods regardless of the contents of the container images.\n\n#### Kubernetes System Images\n\nKubernetes itself is migrating to [distroless for k8s system images] such as\n`kube-apiserver` and `kube-dns`. This led to the creation of a [scratch\ndebugger] to copy debugging tools into the running container, but this script\nhas some downsides:\n\n1. Since it's not possible to install the debugging tools using only Kubernetes,\n   the script issues docker commands directly and so isn't portable to other\n   runtimes.\n2. The script requires broad administrative access to the node to run docker\n   commands.\n3. Installing tools into the running container requires modifying the image\n   being debugged.\n\n`kubectl debug` would replace the scratch debugger script with a method\nidiomatic to Kubernetes.\n\n#### Operations and Support\n\nAs Kubernetes gains in popularity, it's becoming the case that a person\ntroubleshooting an application is not necessarily the person who built it.\nOperations staff and Support organizations want the ability to attach a \"known\ngood\" or automated debugging environment to a pod.\n\n### Goals\n\nMake available to users of Kubernetes a troubleshooting facility that:\n\n1. works out of the box as a first-class feature of the platform.\n2. does not depend on tools having already been included in container images.\n3. does not require administrative access to the node. (Administrative access\n   via the Kubernetes API is acceptable.)\n\nNew `kubectl` concepts increase cognitive burden for all users of Kubernetes.\nThis KEP seeks to minimize this burden by mirroring the existing `kubectl`\nworkflows where possible.\n\n### Non-Goals\n\nEphemeral containers are supported on Windows, but it's not the recommended\ndebugging facility. The other troubleshooting workflows described here are\nequally as useful to Windows containers. We will not attempt to create\nseparate debugging facilities for Windows containers.\n\n[distroless container image]: https://github.com/GoogleCloudPlatform/distroless\n[distroless for k8s system images]: https://git.k8s.io/enhancements/keps/sig-release/20190316-rebase-images-to-distroless.md\n[scratch debugger]: https://github.com/kubernetes-retired/contrib/tree/master/scratch-debugger\n\n## Proposal\n\n### Pod Troubleshooting with Ephemeral Debug Container\n\nWe will add a new command `kubectl debug` that will:\n\n1. Construct a `v1.Container` for the debug container based on command line\n   arguments. This will include an optional container for namespace targeting\n   (described below).\n2. Fetch the specified pod's existing ephemeral containers using\n   `GetEphemeralContainers()` in the generated pod client.\n3. Append the new debug container to the pod's ephemeral containers and call\n   `UpdateEphemeralContainers()`.\n4. Watch pod for updates to the debug container's `ContainerStatus` and\n   automatically attach once the container is running. *(optional based on\n   command line flag)*\n\nWe will attempt to make `kubectl debug` useful with a minimal of arguments\nby using reasonable defaults where possible.\n\n#### Debug Container Naming\n\nCurrently, there is no support for deleting or recreating ephemeral containers.\nIn cases where the user does not specify a name, `kubectl` should generate a\nunique name and display it to the user.\n\n#### Container Namespace Targeting\n\nIn order to see processes running in other containers in the pod, [Process\nNamespace Sharing] should be enabled for the pod. In cases where process\nnamespace sharing isn't enabled for the pod, `kubectl` will set\n`TargetContainer` in the `EphemeralContainer`. This will cause the ephemeral\ncontainer to be created in the namespaces of the target container in runtimes\nthat support container namespace targeting.\n\n[Process Namespace Sharing]: https://kubernetes.io/docs/tasks/configure-pod-container/share-process-namespace\n\n#### Interactive Troubleshooting and Automatic Attaching\n\nSince the primary use case for `kubectl debug` is interactive troubleshooting,\n`kubectl debug` will automatically attach to the console of the newly created\nephemeral container and will default to creating containers with `Stdin` and\n`TTY` enabled.\n\nThese may be disabled via command line flags.\n\n#### Proposed Ephemeral Debug Arguments\n\n```\n% kubectl help debug\nExecute a container in a pod.\n\nExamples:\n  # Start an interactive debugging session with a debian image\n  kubectl debug mypod --image=debian\n\n  # Run a debugging session in the same namespace as target container 'myapp'\n  # (Useful for debugging other containers when shareProcessNamespace is false)\n  kubectl debug mypod --target=myapp\n\nOptions:\n  -a, --attach=true: Automatically attach to container once created\n  -c, --container='': Container name.\n  -i, --stdin=true: Pass stdin to the container\n  --image='': Required. Container image to use for debug container.\n  --target='': Target processes in this container name.\n  -t, --tty=true: Stdin is a TTY\n\nUsage:\n  kubectl debug (POD | TYPE/NAME) [-c CONTAINER] [flags] -- COMMAND [args...] [options]\n\nUse \"kubectl options\" for a list of global command-line options (applies to all commands).\n```\n\n### Pod Troubleshooting by Copy\n\nPod troubleshooting via Ephemeral Container relies on an alpha feature which is\nunlikely to be enabled on production clusters. In order to support these\nclusters, and because it would be generally useful, we will support a mode of\nPod troubleshooting that behaves similar to Pod Troubleshooting with Ephemeral\nDebug Container but operates instead on a copy of the target pod.\n\nThe following additional options will cause a copy of the target pod to be\ncreated:\n\n```\nOptions:\n  --copy-to='': Create a copy of the target Pod with this name.\n  --copy-labels=false: When used with `--copy-to`, specifies whether labels\n                       should also be copied. Note that copying labels may cause\n                       the copy to receive traffic from a service or a replicaset\n                       to kill other Pods.\n  --delete-old=false: When used with `--copy-to`, delete the original Pod.\n  --edit=false: Open an editor to modify the generated Pod prior to creation.\n  --same-node=false: Schedule the copy of target Pod on the same node.\n  --share-processes=true: When used with `--copy-to`, enable process namespace\n                          sharing in the copy.\n```\n\nThe modification `kubectl debug` makes to `Pod.Spec.Containers` depends on the\nvalue of the `--container` flag.\n\n#### Creating a Debug Container by copy\n\nIf a user does not specify a `--container` or specifies one that does not exist,\nthen the user is instructing `kubectl debug` to create a new Debug Container in\nthe Pod copy.\n\n```\nExamples:\n  # Create a copy of 'mypod' with a new debugging container and attach to it\n  kubectl debug mypod --copy-to=mypod-debug --image=debian --attach -- bash\n```\n\n#### Modify Application Image by Copy\n\nIf a user specifies a `--container`, then they are instructing `kubectl debug` to\ncreate a copy of the target pod with a new image for one of the containers.\n\n```\nExamples:\n  # Create a copy of 'mypod' with the debugging image for container 'app'\n  kubectl debug mypod --copy-to=mypod-debug --image=myapp-image:debug --container=myapp -- myapp --debug=5\n```\n\nNote that the Pod API allows updates of container images in-place, so\n`--copy-to` is not necessary for this operation. `kubectl debug` isn't necessary\nto achieve this -- it can be done today with patch -- but `kubectl debug` could\nimplement it as well for completeness.\n\n### Node Troubleshooting with Privileged Containers\n\nWhen invoked with a node as a target, `kubectl debug` will create a new\npod with the following fields set:\n\n* `nodeName: $target_node`\n* `hostIPC: true`\n* `hostNetwork: true`\n* `hostPID: true`\n* `restartPolicy: Never`\n\nAdditionally, `/` on the node will be mounted as a HostPath volume.\n\n```\nExamples:\n  # Start an interactive debugging session on mynode with a debian image\n  kubectl debug node/mynode --image=debian\n\nOptions:\n  -a, --attach=true: Automatically attach to container once created\n  -c, --container='': Container name.\n  -i, --stdin=true: Pass stdin to the container\n  --image='': Required. Container image to use for debug container.\n  -t, --tty=true: Stdin is a TTY\n```\n\n### User Stories\n\n#### Operations\n\nAlice runs a service \"neato\" that consists of a statically compiled Go binary\nrunning in a minimal container image. One of its pods is suddenly having\ntrouble connecting to an internal service. Being in operations, Alice wants to\nbe able to inspect the running pod without restarting it, but she doesn't\nnecessarily need to enter the container itself. She wants to:\n\n1.  Inspect the filesystem of target container\n1.  Execute debugging utilities not included in the container image\n1.  Initiate network requests from the pod network namespace\n\nThis is achieved by running a new \"debug\" container in the pod namespaces. Her\ntroubleshooting session might resemble:\n\n```\n% kubectl debug -it --image debian neato-5thn0 -- bash\nroot@debug-image:~# ps x\n  PID TTY      STAT   TIME COMMAND\n    1 ?        Ss     0:00 /pause\n   13 ?        Ss     0:00 bash\n   26 ?        Ss+    0:00 /neato\n  107 ?        R+     0:00 ps x\nroot@debug-image:~# cat /proc/26/root/etc/resolv.conf\nsearch default.svc.cluster.local svc.cluster.local cluster.local\nnameserver 10.155.240.10\noptions ndots:5\nroot@debug-image:~# dig @10.155.240.10 neato.svc.cluster.local.\n\n; \u003c\u003c\u003e\u003e DiG 9.9.5-9+deb8u6-Debian \u003c\u003c\u003e\u003e @10.155.240.10 neato.svc.cluster.local.\n; (1 server found)\n;; global options: +cmd\n;; connection timed out; no servers could be reached\n```\n\nAlice discovers that the cluster's DNS service isn't responding.\n\n#### Debugging\n\nBob is debugging a tricky issue that's difficult to reproduce. He can't\nreproduce the issue with the debug build, so he attaches a debug container to\none of the pods exhibiting the problem:\n\n```\n% kubectl debug -it --image=gcr.io/neato/debugger neato-5x9k3 -- sh\nDefaulting container name to debug.\n/ # ps x\nPID   USER     TIME   COMMAND\n    1 root       0:00 /pause\n   13 root       0:00 /neato\n   26 root       0:00 sh\n   32 root       0:00 ps x\n/ # gdb -p 13\n...\n```\n\nHe discovers that he needs access to the actual container, which he can achieve\nby installing busybox into the target container:\n\n```\nroot@debug-image:~# cp /bin/busybox /proc/13/root\nroot@debug-image:~# nsenter -t 13 -m -u -p -n -r /busybox sh\n\n\nBusyBox v1.22.1 (Debian 1:1.22.0-9+deb8u1) built-in shell (ash)\nEnter 'help' for a list of built-in commands.\n\n/ # ls -l /neato\n-rwxr-xr-x    2 0        0           746888 May  4  2016 /neato\n```\n\nNote that running the commands referenced above requires `CAP_SYS_ADMIN` and\n`CAP_SYS_PTRACE`.\n\nThis scenario also requires process namespace sharing which is not available\non Windows.\n\n#### Automation\n\nCarol is a security engineer tasked with running security audits across all of\nher company's running containers. Even though her company has no standard base\nimage, she's able to audit all containers using:\n\n```\n% for pod in $(kubectl get -o name pod); do\n    kubectl debug --image gcr.io/neato/security-audit -p $pod /security-audit.sh\n  done\n```\n\n#### Technical Support\n\nDan's team provides support for his company's multi-tenant cluster. He can\naccess the Kubernetes API (as a viewer) on behalf of the users he's supporting,\nbut he does not have administrative access to nodes or a say in how the\napplication image is constructed. When someone asks for help, Dan's first step\nis to run his team's autodiagnose script:\n\n```\n% kubectl debug --image=k8s.gcr.io/autodiagnose nginx-pod-1234\n```\n\n### Implementation Details/Notes/Constraints\n\n1.  There's an output stream race inherent to creating then attaching a\n    container which causes output generated between the start and attach to go\n    to the log rather than the client. This is not specific to Ephemeral\n    Containers and exists because Kubernetes has no mechanism to attach a\n    container prior to starting it. This larger issue will not be addressed by\n    Ephemeral Containers, but Ephemeral Containers would benefit from future\n    improvements or work arounds.\n\n### Risks and Mitigations\n\n1.  There are no guaranteed resources for ad-hoc troubleshooting. If\n    troubleshooting causes a pod to exceed its resource limit it may be evicted.\n    This risk can be removed once support for pod resizing has been implemented.\n\n## Design Details\n\n### Test Plan\n\nIn addition to standard unit tests for `kubectl`, the `debug` command will be\nreleased as a `kubectl alpha` subcommand, signaling users to expect instability.\nDuring the alpha phase we will gather feedback from users that we expect will\nimprove the design of `debug` and identify the Critical User Journeys we should\ntest prior to Alpha -\u003e Beta graduation.\n\n### Graduation Criteria\n\n#### Alpha -\u003e Beta Graduation\n\n- [ ] Ephemeral Containers API has graduated to Beta\n- [ ] A task on https://kubernetes.io/docs/tasks/ describes how to troubleshoot\n  a running pod using Ephemeral Containers.\n- [ ] A survey sent to early adopters doesn't reveal any major shortcomings.\n- [ ] Test plan is amended to address the most common user journeys.\n\n#### Beta -\u003e GA Graduation\n\n- [ ] Ephemeral Containers are GA\n\n## Implementation History\n\n- *2019-08-06*: Initial KEP draft\n- *2019-12-05*: Updated KEP for expanded debug targets.\n- *2020-01-09*: Updated KEP for debugging nodes and mark implementable.\n- *2020-01-15*: Added test plan.\n\n## Alternatives\n\nAn exhaustive list of alternatives to ephemeral containers is included in the\n[Ephemeral Containers KEP].\n\n[Ephemeral Containers KEP]: https://git.k8s.io/enhancements/keps/sig-node/20190212-ephemeral-containers.md\n"
+  },
+  {
+    "id": "97503d66c4b959b1d359eb6e3e29a723",
+    "title": "future-of-kubectl-cp",
+    "authors": ["@sallyom"],
+    "owningSig": "sig-cli",
+    "participatingSigs": ["sig-usability"],
+    "reviewers": ["@liggitt", "@brendandburns"],
+    "approvers": ["@pwittrock", "@soltysh"],
+    "editor": "TBD",
+    "creationDate": "2019-09-20",
+    "lastUpdated": "2019-09-20",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# future-of-kubectl-cp\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals for kubectl cp](#goals-for-kubectl-cp)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [UserA wants to copy a file or a directory from a local host to a pod (or a container in a pod)](#usera-wants-to-copy-a-file-or-a-directory-from-a-local-host-to-a-pod-or-a-container-in-a-pod)\n    - [UserB wants to copy a file or a directory from a pod to a local host](#userb-wants-to-copy-a-file-or-a-directory-from-a-pod-to-a-local-host)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Drawbacks](#drawbacks)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nThis document summarizes and originates from this email thread, \n[Proposal to drop kubectl cp](https://groups.google.com/forum/?utm_medium=email\u0026utm_source=footer#!msg/kubernetes-sig-cli/_zUy67lK49k/aE6vncYiAgAJ).   \n\nThis document aims to solidify the future of `kubectl cp` as a tool that provides basic function of copying files between local environments and pods.  Any advanced use cases\nsuch as those involving symlinks or modifying file permissions should be performed outside of `kubectl cp` through `kubectl exec`, addons, or shell commands.    \n\nOver the past few releases, there have been numerous security issues with `kubectl cp` that have resulted in release updates in all supported versions of kubectl.\nAt the same time,any new PR that extends `kubectl cp` must undergo extra reviews to evaluate security threats that may arise [1][2].  Over the past few months,\nsecurity fixes have required dropping edge cases and function of the command.  It is increasingly difficult to maintain a cp command that is both\nuseful and secure.  There are alternative approaches that provide the same function as `kubectl cp` [3].  Using `kubectl exec ...| tar`\nprovides transparency when copying files as well as mitigations for path traversals, symlink directory escapes, tar bombs, and other exploits.\nUse of tar is more featureful, in that it can preserve file permissions and copy pod-to-pod.  Also, `kubectl cp` is dependent on the tar binary\nin a container.  A malicious tar binary is outside of what `kubectl cp` can control.    \n\nWith all of this in mind the cost and risk of maintaining the cp command should be weighed against what is considered crucial functionality in kubectl. \nIt's better to address 80% of use cases with a simple tool than trying to address the remaining 20% at the cost of risking those 80%.     \n\n[1] https://github.com/kubernetes/kubernetes/pull/78622   \n[2] https://github.com/kubernetes/kubernetes/pull/73053   \n[3] https://gist.github.com/tallclair/9217e2694b5fdf27b55d6bd1fda01b53   \n\n## Motivation\n\n- The `kubectl cp` command has been the subject of multiple security vulnerability reports.\n    * [CVE-2018-1002100](http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-1002100)\n    * [CVE-2019-1002101](http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-1002101)\n    * [CVE-2019-11246](https://cve.mitre.org/cgi-bin/cvename.cgi?name=2019-11246)\n    * [CVE-2019-11249](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-11249)\n- To use `kubectl cp`, container images are required to have the tar binary. `kubectl cp` is not available when running containers from the minimal [scratch image](https://hub.docker.com/_/scratch/).    \n  Running from scratch is by itself a tactic to securing containers, as it encourages the best practice of limiting the tools packaged in an image to only what's required by a workload.   \n\nThis proposal is that `kubectl cp` should perform only basic copying of files.  Advanced features of file copying should be out of scope for `kubectl cp`.  \n\n### Goals for kubectl cp\n\n- Provide simple function to copy a single file or directory, without advanced features such as symlinks or file permission changes\n- Ensure, through extra review process tbd, that features added to `kubectl cp` are well understood and easy to secure.\n- Offer users example `kubectl exec`/ shell commands to address advanced options of copying files. (There are a few already in --help)\n  - As an alternative, problematic `kubectl cp` code (handling symlinks) can be replaced with code to shell out to `kubectl exec ... | tar`\n\n### Non-Goals\n\nFor either of these, a separate proposal weighing the cost/benefit would be required.  These are out of scope of this proposal to simplify `kubectl cp`:\n- Rewrite `kubectl cp` to not use tar, by modifying CRI as outlined partially [here](https://github.com/kubernetes/kubernetes/issues/58512). \n- Rewrite `kubectl cp` to be functional in scratch based containers through use of ephemeral containers as outlined [here](https://github.com/kubernetes/kubernetes/issues/58512#issuecomment-528384746)\n\n## Proposal\n\n- `kubectl cp` should provide simple function of copying single file or directory between local environments and pods.\n- Identify and document `kubectl exec` commands to address more advanced options for copying files.  \n- Provide users attempting to use `kubectl cp + symlinks/etc` with output showing comparable `kubectl exec ...| tar` cmds.\n- It is up for a decision in this proposal whether the community prefers to implement the `shelling out to tar from within kubectl cp` \nor leave as suggestions in error output. \n- Barring decision of the above, only the user stories listed below should be supported by `kubectl cp`.  If additional user stories are added via shelling out to tar from kubectl, \n  those will be outlined below.    \n\n### User Stories\n\n#### UserA wants to copy a file or a directory from a local host to a pod (or a container in a pod)\n\n```console\n $ kubectl cp localdir somens/somepod:/poddir (-c acontainer)\n $ kubectl cp localdir somens/somepod:/poddir (-c acontainer)\n $ kubectl cp localdir/filea somens/somepod:/poddir (-c acontainer)\n```\n\n#### UserB wants to copy a file or a directory from a pod to a local host\n\n```console\n $ kubectl cp somens/somepod:/poddir localdir\n $ kubectl cp somens/somepod:/poddir/filea localdir/filea\n```\n\n### Implementation Details/Notes/Constraints [optional]\n\n### Risks and Mitigations\n\nAny scripts or automation that currently rely on advanced features of `kubectl cp` will be broken.\nTo mitigate, detailed information about why the command now fails as well as example `kubectl exec ...| tar` alternatives will be output. \n\n## Design Details\n\n### Test Plan\n\nInclude test automation script that calls all `kubectl cp` flags or options that are removed.\nEnsure that failure includes example alternative approach, plus information about the failure, skipped files, etc.\n\n### Graduation Criteria\n\n### Upgrade / Downgrade Strategy\n\n`kubectl cp` function removed as a result of a CVE fix or other will be documented clearly.\nInformation about why subcommand/option is no longer supported, what files are skipped, and also alternative `kubectl exec ...| tar` commands \nwill be included in failed command output.  This output will then always be given (not just for a deprecation period). \n\n### Version Skew Strategy\n\n## Implementation History\n\n## Drawbacks\n\nAutomation scripts that include `kubectl cp` will be broken if options and features are removed from the command.\nThe motivation of improving security is weighed against this potential drawback.  \n"
+  },
+  {
+    "id": "c1cf490422f61d695bebfa989c0e9022",
+    "title": "Kubectl Commands In Headers",
+    "authors": ["@pwittrock"],
+    "owningSig": "sig-cli",
+    "participatingSigs": ["sig-api-machinery"],
+    "reviewers": ["@deads2k", "@kow3ns", "@lavalamp"],
+    "approvers": ["@seans3", "@soltysh"],
+    "editor": "TBD",
+    "creationDate": "2019-02-22",
+    "lastUpdated": "2019-02-22",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# kubectl-commands-in-headers\n\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n  - [Anti-Goals](#anti-goals)\n- [Proposal](#proposal)\n  - [X-Kubectl-Command Header](#x-kubectl-command-header)\n  - [X-Kubectl-Flags Header](#x-kubectl-flags-header)\n    - [Enumerated Flag Values](#enumerated-flag-values)\n  - [X-Kubectl-Session Header](#x-kubectl-session-header)\n  - [X-Kubectl-Deprecated Header](#x-kubectl-deprecated-header)\n  - [X-Kubectl-Build Header](#x-kubectl-build-header)\n  - [Example](#example)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n  - [#859](https://github.com/kubernetes/enhancements/issues/859)\n- [x] KEP approvers have set the KEP status to `implementable`\n- [x] Design details are appropriately documented\n- [x] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n  - Standard Unit and Integration testing should be sufficient\n- [x] Graduation criteria is in place\n  - This is not a user facing API change\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nRequests sent to the apiserver from kubectl already include a User Agent header with\ninformation about the kubectl build.  This KEP proposes sending http request headers\nwith additional context about the kubectl command that created the request.\nThis information may be used by operators of clusters for debugging or\nto gather telemetry about how users are interacting with the cluster.\n\n\n## Motivation\n\nKubectl generates requests sent to the apiserver for commands such as `apply`, `delete`, `edit`, `run`, however\nthe context of the command for the requests is lost and unavailable to cluster administrators.  Context would be\nuseful to cluster admins both for debugging the cause of requests as well as providing telemetry about how users\nare interacting with the cluster, which could be used for various purposes.\n\n### Goals\n\n- Allow cluster administrators to identify how requests in the logs were generated from\n  kubectl commands.\n  \nPossible applications of this information may include but are not limited to:\n\n- Organizations could learn how users are interacting will their clusters to inform what internal\n  tools they build and invest in or what gaps they may need to fill.\n- Organizations could identify if users are running deprecated commands that will be removed\n  when the version of kubectl is upgraded.  They could do this before upgrading kubectl.\n  - SIG-CLI could build tools that cluster admins run and perform this analysis\n    to them to help with understanding whether they will be impacted by command deprecation\n- Organizations could identify if users are running kubectl commands that are inconsistent with \n  the organization's internal best practices and recommendations.\n- Organizations could voluntarily choose to bring back high-level learnings to SIG-CLI regarding\n  which and how commands are used.  This could be used by the SIG to inform where to invest resources\n  and whether to deprecate functionality that has proven costly to maintain.\n- Cluster admins debugging odd behavior caused by users running kubectl may more easily root cause issues\n  (e.g. knowing what commands were being run could make identifying miss-behaving scripts easier)\n- Organizations could build dashboards visualizing which kubectl commands where being run\n  against clusters and when.  This could be used to identify broader usage patterns within the\n  organization.\n\n\n### Non-Goals\n\n*The following are not goals of this KEP, but could be considered in the future.*\n\n- Supply Headers for requests made by kubectl plugins.  Enforcing this would not be trivial.\n- Send Headers to the apiserver for kubectl command invocations that don't make requests - \n  e.g. `--dry-run`\n\n### Anti-Goals\n\n*The following should be actively discouraged.*\n\n- Make decisions of any sort in the apiserver based on these headers.\n  - This information is intended to be used by humans for the purposes of developing a better understanding\n    of kubectl usage with their clusters, such as **for debugging and telemetry**.\n\n## Proposal\n\nInclude in http requests made from kubectl to the apiserver:\n\n- the kubectl subcommand\n- which flags were specified as well as whitelisted enum values for flags (never arbitrary values)\n- a generated session id\n- never include the flag values directly, only use a predefined enumeration\n- never include arguments to the commands, only the sub commands themselves\n- if the command is deprecated, add a header including when which release it will be removed in (if known)\n- allow users and organizations that compile their own kubectl binaries to define a build metadata header\n\n### X-Kubectl-Command Header\n\nThe `X-Kubectl-Command` Header contains the kubectl sub command.\n\nIt contains the path to the subcommand (e.g. `create secret tls`) to disambiguate sub commands\nthat might have the same name and different paths.\n\nExamples:\n\n- `X-Kubectl-Command: apply`\n- `X-Kubectl-Command: create secret tls` \n- `X-Kubectl-Command: delete`\n- `X-Kubectl-Command: get`\n\n### X-Kubectl-Flags Header\n\nThe `X-Kubectl-Flags` Header contains a list of the kubectl flags that were provided to the sub\ncommand.  It does *not* contain the raw flag values, but may contain enumerations for\nwhitelisted flag values.  (e.g. for `-f` it may contain whether a local file, stdin, or remote file\nwas provided).  It does not normalize into short or long form.  If a flag is\nprovided multiple times it will appear multiple times in the list.  Flags are sorted\nalpha-numerically and separated by a ',' to simplify human readability.\n\nExamples:\n\n- `X-Kubectl-Flags: --filename=local,--recursive,--context`\n- `X-Kubectl-Flags: -f=local,-f=local,-f=remote,-R` \n- `X-Kubectl-Flags: -f=stdin` \n- `X-Kubectl-Flags: --dry-run,-o=custom-columns`\n\n#### Enumerated Flag Values\n\n- `-f`, `--filename`: `local`, `remote`, `stdin`\n- `-o`, `--output`: `json`,`yaml`,`wide`,`name`,`custom-columns`,`custom-columns-file`,`go-template`,`go-template-file`,`jsonpath`,`jsonpath-file`\n- `--type` (for patch subcommand): `json`, `merge`, `strategic`\n\n### X-Kubectl-Session Header\n\nThe `X-Kubectl-Session` Header contains a Session ID that can be used to identify that multiple\nrequests which were made from the same kubectl command invocation.  The Session Header is generated\nonce and used for all requests for each kubectl process.\n\n- `X-Kubectl-Session: 67b540bf-d219-4868-abd8-b08c77fefeca`\n\n### X-Kubectl-Deprecated Header\n\nThe `X-Kubectl-Deprecated` Header is set to inform cluster admins that the command being run\nhas been deprecated.  This may be used by organizations to determine if they are likely\nto be impacted by command deprecation and removal before they upgrade.\n\nThe `X-Kubectl-Deprecated` Header is set if the command that was run is marked as deprecated.\n\n- The Header may have a value of `true` if the command has been deprecated, but has no removal date.\n- The Header may have a value of a specific Kubernetes release.  If it does, this is the release\n  that the command will be removed in.\n\n- `X-Kubectl-Deprecated: true`\n- `X-Kubectl-Deprecated: v1.16`\n\n\n### X-Kubectl-Build Header\n\nThe `X-Kubectl-Build` Header may be set by building with a specific `-ldflags` value.  By default the Header\nis unset, but may be set if kubectl is built from source, forked, or vendored into another command.\nOrganizations that distribute one or more versions of kubectl which they maintain internally may\nset a flag at build time and this header will be populated with the value.\n\n- `X-Kubectl-Build: some-value`\n\n### Example\n\n```sh\n$ kubectl apply -f - -o yaml\n```\n\n```\nX-Kubectl-Command: apply\nX-Kubectl-Flags: -f=stdin,-o=yaml\nX-Kubectl-Session: 67b540bf-d219-4868-abd8-b08c77fefeca\n```\n\n\n```sh\n$ kubectl apply -f ./local/file -o=custom-columns=NAME:.metadata.name\n```\n\n```\nX-Kubectl-Command: apply\nX-Kubectl-Flags: -f=local,-o=custom-columns\nX-Kubectl-Session: 0087f200-3079-458e-ae9a-b35305fb7432\n```\n\n```sh\nkubectl patch pod valid-pod --type='json' -p='[{\"op\": \"replace\", \"path\": \"/spec/containers/0/image\", \"value\":\"new\nimage\"}]'\n```\n\n```\nX-Kubectl-Command: patch\nX-Kubectl-Flags: --type=json,-p\nX-Kubectl-Session: 0087f200-3079-458e-ae9a-b35305fb7432\n```\n\n\n```sh\nkubectl run nginx --image nginx\n```\n\n```\nX-Kubectl-Command: run\nX-Kubectl-Flags: --image\nX-Kubectl-Session: 0087f200-3079-458e-ae9a-b35305fb7432\nX-Kubectl-Deprecated: true\n```\n\n### Risks and Mitigations\n\nUnintentionally including sensitive information in the request headers - such as local directory paths\nor cluster names.  This won't be a problem as the command arguments and flag values are never directly\nincluded.\n\n## Design Details\n\n### Test Plan\n\n- Verify the Command Header is sent for commands and has the correct value\n- Verify the Flags Header is sent for flags and has the correct value\n- Verify the Session Header is sent for the Session and has a legitimate value\n- Verify the Deprecation Header is sent for the deprecated commands and has the correct value\n- Verify the Build Header is sent when the binary is built with the correct ldflags value\n  specified and has the correct value\n\n### Graduation Criteria\n\n- Determine if additional information would be valuable to operators of clusters.\n- Consider building and publishing tools for cluster operators to run which make use of the data\n  - Look for deprecated command invocations\n  - Build graphs of usage\n  - Identify most used commands\n\n### Upgrade / Downgrade Strategy\n\nNA\n\n### Version Skew Strategy\n\nNA\n\n## Implementation History\n\n\n"
+  },
+  {
+    "id": "40ae618e5503cc80c0251871759e40c7",
+    "title": "Move Kubectl Code into Staging",
+    "authors": ["@seans3"],
+    "owningSig": "sig-cli",
+    "participatingSigs": null,
+    "reviewers": ["@pwittrock", "@liggitt"],
+    "approvers": ["@soltysh"],
+    "editor": "@seans3",
+    "creationDate": "2019-04-09",
+    "lastUpdated": "2019-04-09",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n\n# Move Kubectl Code into Staging\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Adding the staging repository in kubernetes/kubernetes:](#adding-the-staging-repository-in-kuberneteskubernetes)\n  - [Modify and Set-up the existing receiving kubernetes/kubectl repository](#modify-and-set-up-the-existing-receiving-kuberneteskubectl-repository)\n  - [Move \u003ccode\u003epkg/kubectl\u003c/code\u003e Code](#move--code)\n  - [Timeframe](#timeframe)\n- [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Testing](#testing)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nWe propose to move the `pkg/kubectl` code base into staging. This effort would\nentail moving `k8s.io/kubernetes/pkg/kubectl` to an appropriate location under\n`k8s.io/kubernetes/staging/src/k8s.io/kubectl`. The `pkg/kubectl`\ncode would not have to move all at once; it could be an incremental process. When\nthe code is moved, imports of the `pkg/kubectl` code would be updated to the\nanalogous `k8s.io/kubectl` imports.\n\n## Motivation\n\nMoving `pkg/kubectl` to staging would provide at least three benefits:\n\n1. SIG CLI is in the process of moving the kubectl binary into its own repository\nunder https://github.com/kubernetes/kubectl. Moving `pkg/kubectl` into staging\nis a necessary first step in this kubectl independence effort.\n\n2. Over time, kubectl has grown to inappropriately depend on various internal\nparts of the Kubernetes code base, creating tightly coupled code which is\ndifficult to modify and maintain. For example, internal resource types (those\nthat are used as the hub in type conversion) have escaped the API Server and\nhave been incorporated into kubectl. Code in the staging directories can not\nhave dependencies on internal Kubernetes code. Moving `pkg/kubectl` to staging would\nprove that `pkg/kubectl` does not contain internal Kubernetes dependencies, and it\nwould permanently decouple kubectl from these dependencies.\n\n3. Currently, it is difficult for external or out-of-tree projects to reuse code\nfrom kubectl packages. The ability to reuse kubectl code outside of\nkubernetes/kubernetes is a long standing request. Moving code to staging\nincrementally unblocks this use case. We will be publish the code from staging\nunder https://github.com/kubernetes/kubectl/ .\n\n### Goals\n\n- Structure kubectl code to eventually move the entire kubectl code base into\n  its own repository.\n- Decouple kubectl permanently from inappropriate internal Kubernetes dependencies.\n- Allow kubectl code to easily be imported into other projects.\n\n### Non-Goals\n\n- Explaining the entire process for moving kubectl into its own repository will\nnot be addressed in this KEP; it will be the subject of its own KEP.\n\n## Proposal\n\nThe following steps to create the staging repository have been copied from\nthe base [staging directory README](https://github.com/kubernetes/kubernetes/tree/master/staging).\n\n### Adding the staging repository in kubernetes/kubernetes:\n\n1. Send an email to the SIG Architecture mailing list and the mailing list of\n   the SIG CLI which would own the repo requesting approval for creating the\n   staging repository.\n\n2. Once approval has been granted, create the new staging repository.\n\n3. Add a symlink to the staging repo in vendor/k8s.io.\n\n4. Update import-restrictions.yaml to add the list of other staging repos that\n   this new repo can import.\n\n5. Add all mandatory template files to the staging repo as mentioned in\n   https://github.com/kubernetes/kubernetes-template-project.\n\n6. Make sure that the .github/PULL_REQUEST_TEMPLATE.md and CONTRIBUTING.md files\n   mention that PRs are not directly accepted to the repo.\n\n### Modify and Set-up the existing receiving kubernetes/kubectl repository\n\nCurrently, there are three types of content in the current kubernetes/kubectl\nrepository that need to be dealt with. These items are 1) the [Kubectl\nBook](https://kubectl.docs.k8s.io/), 2) an integration test framework, and 3)\nsome kubectl openapi code. Since we intend to copy the staging code into this\nkubernetes/kubectl repository, and since this repository must be empty, we need\nto dispose of these items. A copy of the kubectl openapi code already exists\nunder `pkg/kubectl` in the kubernetes/kubernetes repository, so it can be\ndeleted. The following steps describe how we intend to modify the existing\nkubernetes/kubectl repository:\n\n1. We will create a backup of the entire [current\n   kubectl](https://github.com/kubernetes/kubectl) repository in the [Google\n   Cloud Source Repository](https://cloud.google.com/source-repositories/).\n\n2. The [Kubectl Book](https://kubectl.docs.k8s.io/) should be in its own\n   repository. So we will create a new repository, and copy this content into\n   the new repository.\n\n3. We will then clear the [current\n   kubectl](https://github.com/kubernetes/kubectl) repository.\n\n4. Setup branch protection and enable access to the stage-bots team by adding\n   the repo in prow/config.yaml. See #kubernetes/test-infra#9292 for an\n   example.\n\n5. Once the repository has been cleared, update the publishing-bot to publish\n   the staging repository by updating:\n\n   rules.yaml: Make sure that the list of dependencies reflects the staging\n   repos in the go modules.\n\n   fetch-all-latest-and-push.sh: Add the staging repo in the list of repos to be\n   published.\n\n6. Add the staging and published repositories as a subproject for the SIG that\n   owns the repos in sigs.yaml.\n\n7. Add the repo to the list of staging repos in this README.md file.\n\n8. We will re-introduce the integration test framework into the kubectl\n   repository by submitting into the new staging directory.\n\n### Move `pkg/kubectl` Code\n\nMove `k8s.io/kubernetes/pkg/kubectl` to a location under the new\n`k8s.io/kubernetes/staging/src/k8s.io/kubectl` directory, and update all\n`k8s.io/kubernetes/pkg/kubectl` imports. This can be an incremental process.\n\n### Timeframe\n\nThere are three remaining Kubernetes core dependencies that have to be resolved\nbefore all of `pkg/kubectl` can be moved to staging. While we work on those\nremaining dependencies, we can move some `pkg/kubectl` code to staging that is\ncurrently being requested by other projects. Specifically, we will move\n`pkg/kubectl/cmd/apply` into staging as soon as possible. The rest of the code\nwould be moved over the next two releases (1.16 and 1.17).\n\n## Risks and Mitigations\n\nIf a project vendors Kubernetes to import kubectl code, this will break them.\nOn the bright side, afterwards, these importers will have a much cleaner path to \ninclude kubectl code. Before moving forward with this plan, we will identify and\ncommunicate to these projects.\n\n## Graduation Criteria\n\nSince this \"enhancement\" is not a traditional feature, and it provides no new\nfunctionality, graduation criteria does not apply to this KEP.\n\n## Testing\n\nExcept for kubectl developers, this change will be mostly transparent. There is\nno new functionality to test; testing will be accomplished through the current\nunit tests, integration tests, and end-to-end tests.\n\n## Implementation History\n\nSee [kubernetes/kubectl#80](https://github.com/kubernetes/kubectl/issues/80) as\nthe umbrella issue to see the details of the kubectl decoupling effort. This\nissue has links to numerous pull requests implementing the decoupling.\n"
+  },
+  {
+    "id": "8694e41b8e1a42951f9132b4d4960130",
+    "title": "Kustomize Exec Secret Generator",
+    "authors": ["@pwittrock"],
+    "owningSig": "sig-cli",
+    "participatingSigs": null,
+    "reviewers": ["@anguslees", "@Liujingfang1", "@sethpollack"],
+    "approvers": ["@monopole"],
+    "editor": "@pwittrock",
+    "creationDate": "2019-03-12",
+    "lastUpdated": "2019-03-12",
+    "status": "implementable",
+    "seeAlso": [
+      "https://github.com/kubernetes/enhancements/blob/master/keps/sig-cli/kustomize-secret-generator-plugins.md"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n\n# Kustomize Exec Secret Generator\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [New Flags](#new-flags)\n  - [Example](#example)\n    - [Default Path](#default-path)\n    - [Flag Defined Absolute Path](#flag-defined-absolute-path)\n    - [Flag Defined Relative Path](#flag-defined-relative-path)\n    - [Not Enabled](#not-enabled)\n- [Risks and Mitigations](#risks-and-mitigations)\n  - [Security](#security)\n- [Alternatives Considered](#alternatives-considered)\n  - [Git Style Plugins](#git-style-plugins)\n    - [Enable Plugins with Environment Variables](#enable-plugins-with-environment-variables)\n- [Graduation Criteria](#graduation-criteria)\n  - [Graduation to GA](#graduation-to-ga)\n    - [Audit Command](#audit-command)\n    - [More OS Specific install locations](#more-os-specific-install-locations)\n- [Testing and documentation.](#testing-and-documentation)\n  - [Testing](#testing)\n  - [Documentation](#documentation)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThe ability to generate Secrets using `exec` was removed in kustomize v2 because of security concerns\nabout users kustomizing malicious `kustomization.yaml`s and thereby providing a path for `kustomization.yaml`'s\npublishers to execute arbitrary commands on the machines of any user who applies the `kustomization.yaml`.\n\nExample goal to enable:\n\n- Alice wants to develop an Application requiring a shared Secret, and to deploy it on Kubernetes using GitOps\n- Alice wants her GitOps deployment mechanism to pull the Secrets that it deploys from an\n  remote source without writing the Secrets as files to local disk.\n- Alice's organization configures the gitops deployment container to run Kustomize in the cluster\n  and be capable of pulling Secrets from remote locations\n- Alice writes her kustomization.yaml to use the generation options configured by her organization.\n\nExample exploit to avoid:\n\n- Alice wants to run a whitebox mysql instance on a test cluster\n- Chuck publishes a whitebox mysql `kustomization.yaml` on GitHub, with a SecretGenerator\n  that will read Alice's ~/.kube/config and send it to Chuck's server by executing `sh`\n  will run a script to generate some Secret\n- Alice runs `kubectl apply -k https://github.com/chuck/mysql` and has the credentials\n  of all of her Kubernetes clusters sent to Chuck when the Secret is generated.\n\nSee [kubernetes-sigs/kustomize#692](https://github.com/kubernetes-sigs/kustomize/issues/692) for more details.\n\n## Motivation\n\nThe ability to create Kubernetes Secrets generated by commands is commonly requested by users.\nThis is useful for use cases where the user does not want the Secrets to be appear decrypted\non disk before being applied to the cluster.\n\nExamples:\n\n- Secrets sources are encrypted and need to be decrypted before they are applied\n- Secrets sources are not stored locally, and need to be fetched from some remote location\n- Secrets are generated using some function (e.g. private-keys)\n\n**Note:** For the target case, *the command for generating the Secret already exists as an executable on\nthe user's machine*.  User's are not expected to want a marketplace of solutions, rather instead they are\nexpected to want to be able to invoke the tools they already use for addressing this task.\n\n### Goals\n\n- Enable users to generate Secrets using the tools they already use to do so\n- Secure by default - Alice must configure her environment in an insecure manner and run the command in an \n  insecure way for it to be exploitable\n- Support Linux / Mac / Windows OS's\n\n### Non-Goals\n\n- Support an ecosystem of users (i.e. not centralized within a single organization) authoring plugins\n  and sharing them with one another.\n- Eliminate all friction for publicly published whitebox `kustomization.yaml`s to generate Secrets\n\n## Proposal\n\nRe-introduce `exec` SecretGenerators, but with the following safeguards in place to address security concerns.\n\n- Plugins must be explicitly enabled with a flag.  If it is not enabled and is used, Kustomize will exit 1 without\n  running exec.\n  - This is to protect against the case where exploitable plugins are installed and used safely internally in an\n    organization.\n  - If exiting 1, Kustomize will first print the list of plugins exec commands that were not executed, and a message\n    about the flag.\n- Executables are restricted to flag defined paths - defaulted a location that is empty by default.  Users must install / symlink executables to this location.\n\n### New Flags\n\n|Feature   | Name               | Type     | Default                                         |  Description |\n| `enabled`| `--enable-kust-plugins` | bool     | `false`                                         | Enable plugins.  If set to `false` and plugins are specified by the `kustomization.yaml` (recursively), then Kustomize will print the plugins that would be run and exit 1. |\n| `path`   | `--kust-plugin-path`    | []string | [\"`$XDG_CONFIG_HOME/kustomize/plugins/kvSources`\", \"`/usr/local/kustomize/plugins/kvSources`\"] | List of relative or absolute paths to search for plugin actuators - e.g. for `exec` look for executables on these paths.  For `go-plugins` look for `.so` files on these paths.  Relative paths must are relative to the `kustomization.yaml` provided to the command.  They are never not relative to the base `kustomization.yaml`s. |\n\n**Note:** These flags were chosen so that they are clear when kustomize is embedded in other tools and\nso that it isn't confused with plugins for those tools - e.g. `kubect apply -k`.\n\n### Example\n\n#### Default Path\n\nNote: `my-exec-plugin` exists at `$XDG_CONFIG_HOME/kustomize/plugins/kvSources/my-exec-plugin`\n\n```yaml\nsecretGenerator:\n- name: fromPlugins\n  kvSources:\n  - pluginType: exec\n    name: my-exec-plugin\n    args:\n    - anotherArg\n    - yetAnotherArg\n```\n\n```sh\n$ kustomize build ./ --enable-kust-plugins\n```\n\n#### Flag Defined Absolute Path\n\nNote: `my-exec-plugin` exists at `/foo/kvSources/my-exec-plugin`\n\n```yaml\nsecretGenerator:\n- name: fromPlugins\n  kvSources:\n  - pluginType: exec\n    name: my-exec-plugin\n    args:\n    - anotherArg\n    - yetAnotherArg\n```\n\n```sh\n$ kustomize build ./ --enable-kust-plugins --kust-plugin-path=/foo/kvSources/\n```\n\n#### Flag Defined Relative Path\n\nNote: `my-exec-plugin` exists at `./my-exec-plugin`\n\n```yaml\nsecretGenerator:\n- name: fromPlugins\n  kvSources:\n  - pluginType: exec\n    name: my-exec-plugin\n    args:\n    - anotherArg\n    - yetAnotherArg\n```\n\n```sh\n$ kustomize build ./ --enable-kust-plugins --kust-plugin-path=./\n```\n\n#### Not Enabled\n\nNote: `my-exec-plugin` exists at `/usr/local/kustomize/plugins/kvSources/my-exec-plugin`\n\n```yaml\nsecretGenerator:\n- name: fromPlugins\n  kvSources:\n  - pluginType: exec\n    name: my-exec-plugin\n    args:\n    - anotherArg\n    - yetAnotherArg\n```\n\n```sh\n$ kustomize build ./\n```\n\nOutput:\n\n```bash\nsecretGenerator plugins used, but plugins not enabled.  To enable plugins for trusted sources, specify --enable-kust-plugins.\nsecretGenerator plugins that will be run with --enable-kust-plugins:\n$XDG_CONFIG_HOME/kustomize/plugins/builtin/my-exec-plugin anotherAg yetAnotherARg\n```\n\n## Risks and Mitigations\n\n### Security\n\n**Risk:** Chuck is able to exploit this feature to do something bad on Alice's machine\n\nRequired steps to exploit:\n\n- Alice executes (via `-k` or `kustomize`) a malicious `kustomization.yaml`\n  - Alice does *not* run *without* enabling plugins with `--enable-kust-plugins` to see which\n    plugins will be run and with which values.\n- Alice opted-in to enable plugins by providing the flag `--enable-kust-plugins`\n- Alice or her organization installed the targeted SecretGenerator to `$XDG_CONFIG_HOME/kustomize/plugins/kvSources`\n  or another location she provided with `--kust-plugin-path=\u003cpath-including-exploitable-binary\u003e`\n- The specified SecretGenerator must be exploitable\n  - Simple transformation functions would not be exploitable.\n  - e.g. A command like `cat` would not be possible for Chuck to exploit in a meaningful way.\n\nExample Exploit Commands:\n\nIn kustomize:\n\n```sh\n# User enables the plugin and allows it to exec sh\n$ kustomize build https://github.com/chuck/maliciousapp --enable-kust-plugins --kust-plugin-path=/bin\n```\n\nIn kubectl:\n\n```sh\n# User enables the plugin and allows it to exec sh\n$ kubectl apply -k https://github.com/chuck/maliciousapp --enable-kust-plugins --kust-plugin-path=/bin\n```\n\nAnalysis:\n\nThis is a low risk profile.  Alice has to: kustomize a malicious kustomization file,\nexplicitly enable plugins as command line arguments, and install an exploitable plugin to\nexploit the plugin system.\n\nBy default kustomize will not-execute the plugins, and print out which plugins were specified\nwith their arguments.\n\n## Alternatives Considered\n\n### Git Style Plugins\n\nCreate a plugin mechanism similar to git or kubectl plugins which supports sub commands.\n\nkubectl and git plugins are targeted at providing a clean interactive UX with extensible sub commands.\nBecause the plugins will be invoked declaratively rather than imperatively support for this sort of\nUX is not necessary.\n\n#### Enable Plugins with Environment Variables\n\nAllow users to specify `export KUSTOMIZE_ENABLE_PLUGINS=true` instead of providing the flag.\n\nSince this could be set in the environment, it doesn't provide much additional security over\nthe plugin directories.\n\n\n## Graduation Criteria\n\nTarget Launch Status: Beta\n\nThis is an integration of the previous execution SecretGenerator mechanism, and as\nsuch is relatively well understood.\n\n### Graduation to GA\n\nGather user feedback.  Determine if there are addressable gaps.\n\nConsider the following:\n\n#### Audit Command\n\nAdd a new command in Kustomize: `kustomize audit dir/`\n\nThis will print out which commands (including args and flags) will be invoked when Kustomize is run,\nwithout actually invoking them.  This will generally be helpful for users to view how secrets are generated.\n\n#### More OS Specific install locations\n\nConsider defaulting search path to include OS specific install locations such as `/Library/` or `~/Library` on OS X.\n\n## Testing and documentation.\n\n### Testing\n\nTBD.\n\n### Documentation\n\nTBD\n\n## Implementation History\n\n(TODO add PR's here)\n"
+  },
+  {
+    "id": "b0ad684658d5dc672bd61654f33b2d93",
+    "title": "Extend Kustomize Patches to Multiple Targets",
+    "authors": ["@Liujingfang1"],
+    "owningSig": "sig-cli",
+    "participatingSigs": ["sig-apps"],
+    "reviewers": ["@pwittrock", "@mengqiy"],
+    "approvers": ["@monopole"],
+    "editor": "@Liujingfang1",
+    "creationDate": "2019-03-14",
+    "lastUpdated": "2019-03-18",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": ["n/a"],
+    "markdown": "\n# Extend Kustomize Patches to Multiple Targets\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Docs](#docs)\n- [Test plan](#test-plan)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Summary\nCurrently, there are different types of patches supported in Kustomize:\n[strategic merge patch](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-api-machinery/strategic-merge-patch.md) and [JSON patch](https://tools.ietf.org/html/rfc6902).\n\n```\npatchesStrategicMerge:\n- service_port_8888.yaml\n- deployment_increase_replicas.yaml\n- deployment_increase_memory.yaml\n\npatchesJson6902:\n- target:\n    version: v1\n    kind: Deployment\n    name: my-deployment\n  path: add_init_container.yaml\n- target:\n    version: v1\n    kind: Service\n    name: my-service\n  path: add_service_annotation.yaml\n```\n\nBoth types need group, version, kind and name(GVKN) of a Kubernetes resource to find\nthe unique target to perform the patching. In strategic merge patch, GVKN is included\nin the patch itself. In JSON patch, the GVKN is specified in `kustomization.yaml`.\n\nThere have been [requests](https://github.com/kubernetes-sigs/kustomize/issues/720) for patching multiple targets by one patch for different purposes: \n- override one field for all objects of one type\n- add or remove common command arguments for all containers\n- inject a [sidecar proxy](https://istio.io/docs/setup/kubernetes/sidecar-injection/) as in istio to all containers\n\n## Motivation\n\nExtend current patching mechanism of strategic merge patch and JSON patch from one target\nto multiple targets.\n\n### Goals\n\nAllow users to patch multiple target resources by one patch in Kustomize.\nThe target resources can be matched by the intersection of resources selected by\n- LabelSelector\n- Annotations\n- Group, Version, Kind and Name(Name can be regex)\n\n\n### Non-Goals\n- Add a different type of patches\n\n## Proposal\n\nAdd field to `kustomization.yaml`: `patches`. It has a block specifying the targets,\na relative file path pointing to the patch file, and a type string specifying either\nusing strategic merge patch or JSON patch.\n\n```go\n\ntype Kustomization struct {\n\t// ...\n\n\tPatches []Patch `json:\"patches\"`\n\t// ...\n}\n\ntype Patch struct {\n\t// Path is a relative file path to the patch file.\n\tPath string `json:\"path,omitempty\"`\n\n\t// Target points to the resources that the patch is applied to\n\tTarget PatchTarget `json:\"target,omitempty\"`\n\n\t// Type is one of `StrategicMergePatch` or `JsonPatch`\n\tType string `json:\"type,omitempty\"`\n}\n\n// PatchTarget specifies a set of resources\ntype PatchTarget struct {\n\t// Group of the target\n\tGroup string `json:\"group,omitempty\"`\n\n\t// Version of the target\n\tVersion string `json:\"version,omitemtpy\"`\n\n\t// Kind of the target\n\tKind string `json:\"kind,omitempty\"`\n\n\t// Name of the target\n\t// The name could be with wildcard to match a list of Resources\n\tName string `json:\"name,omitempty\"`\n\n\t// MatchAnnotations is a map of key-value pairs.\n\t// A Resource matches it will be appied the patch\n\tMatchAnnotations map[string][string] `json:\"matchAnnotations,omitempty\"`\n\n\t// LabelSelector is a map of key-value pairs.\n\t// A Resource matches it will be applied the patch.\n\tLabelSelector map[string][string] `json:\"labelSelector,omitempty\"`\n}\n```\n\n**Example:**\n\n```yaml\npatches:\n- file: path/to/patch.yaml\n  type: StrategicMergePatch\n  target:\n    matchAnnotations:\n      foo: bar\n- file: path/to/patch2.yaml\n  type: JsonPatch\n  target:\n    labelSelector:\n      app: test\n- file: path/to/patch3.yaml\n  type: JsonPatch\n  target:\n    Kind: Deployment\n    Name: app1-*\n    labelSelector:\n      env: dev\n- file: path/to/patch4.yaml\n```\n\n### Risks and Mitigations\nThis change is compatible with Kustomize 2.0.*,\nbut need to bump the minor version for feature implementation.\n\n## Graduation Criteria\nSince the proposed `patches` can cover current `patchesJson6902` and\n`patchesStrategicMerge`, those two fields can be deprecated in\nKustomize 3.0.0.\n\n## Docs\n\nUpdate Kustomize and Kubectl docs with this new capability.\n\n## Test plan\n\nUnit test matching Resources and performing customizations.\n\n## Implementation History\n- Add `Patch` struct in `Kustomization` type.\n- Update the patching transformer to recognize `Patch` and match\n  multiple resources\n- Add unit test and integration test\n\n## Alternatives\n"
+  },
+  {
+    "id": "1d84c3a86b85334e69a458f1f37b3b34",
+    "title": "Kustomize File Processing Integration",
+    "authors": ["@pwittrock"],
+    "owningSig": "sig-cli",
+    "participatingSigs": ["sig-cli"],
+    "reviewers": ["@liggitt", "@seans3", "@soltysh", "@monopole"],
+    "approvers": ["@liggitt", "@seans3", "@soltysh", "@monopole"],
+    "editor": "@pwittrock",
+    "creationDate": "2019-01-17",
+    "lastUpdated": "2019-03-18",
+    "status": "implemented",
+    "seeAlso": ["kustomize-subcommand-integration.md"],
+    "replaces": null,
+    "supersededBy": ["n/a"],
+    "markdown": "\n# Kustomize File Processing Integration\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Specifics](#specifics)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n  - [Evaluate and decide](#evaluate-and-decide)\n  - [Implement](#implement)\n- [Docs](#docs)\n- [Test plan](#test-plan)\n  - [Version Skew Tests](#version-skew-tests)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis is a follow up to [KEP Kustomize Subcommand Integration](kustomize-subcommand-integration.md)\n\n[Kustomize](https://github.com/kubernetes-sigs/kustomize) was introduced as \nsubcommand of kubectl to allow users to build their kustomizations directly.\nHowever users need to pipe the kustomize output to other commands in order\nto use the kustomizations.\n\nThis KEP proposes integrating the kustomization libraries into the cli-runtime\nfile processing libraries.  Doing so will provide a cleaner, simpler UX\nand provide a path for addressing issues around error handling and messaging.\n\n## Motivation\n\n- It is capable of removing friction that requires deeper integration - such as producing errors referencing line\n  numbers of the original files (rather than the output files) and exiting with the right error code if kustomization\n  fails.\n- It works for tools that wrap and exec kubectl or vendor kubectl without additional steps\n- It is more consistent with the UX workflow of other commands and flags (other commands don't require pipes)\n- It has a cleaner and simpler UX than requiring a pipe - fewer characters to type\n- It is clear which commands it that support it - apply, get, delete, etc.\n- It can be more clear in the documentation when running `--help` (e.g. the -k flag is shown)\n\n### Goals\n\n- Provide a clean and integrated user experience when working with files from kubectl.\n- Provide consistent UX across kubectl commands for working with kustomized applications.\n\n### Non-Goals\n\n## Proposal\n\nIntegrate kustomize directly into libraries that enable file processing for cli-runtime (e.g. resource builder).\nKubectl commands taking the common flags (`-f`, `--filename`, `-R`, `--recursive`) will support `kustomization.yaml`\nfiles through a new flag `-k` or `--kustomize`.\n\nCli-runtime will add the flags `-k, --kustomize`, which will be registered along side the other file processing\nflags.  If the `-k` flag is provided to a command, the experience will be similar to if the user had piped\nkustomize to stdin - e.g. `kubectl kustomize \u003cvalue\u003e | kubectl \u003ccommand\u003e -f -`.  It will differ in that it provides\nimproved error handling and messaging.\n\nExample: `kubectl apply -k \u003cdir-containing-kustomization\u003e`\n\nTools outside kubectl that use the cli-runtime to register file processing flags and build resources will get the\n`-k` by default, but can opt-out if  they do not want the functionality.\n\n### Specifics\n\n- The `-f` and `-k` flags will initially be mutually exclusive\n- The `-k` flag can be specified at most once\n- The `-k` flag can only point to a *directory* or url containing a file named `kustomization.yaml` file\n  (same as `kustomize`)\n\n### Risks and Mitigations\n\nLow:\n\nWhen run against a `kustomization.yaml` with multiple bases, kubectl may perform multiple requests as part of the\npreprocessing.  Since `-k` is a separate flag from `-f`, it is transparent to a user whether they are running\nagainst a kustomization file or a directory of Resource Config.\n\n## Graduation Criteria\n\nNote: The flag itself does not have an alpha, beta, ga version.  Graduation is taken to mean - proposed iterative\nimprovements to the functionality.\n\nGraduation criteria for the `-k, --kustomize` flag\n\n### Evaluate and decide\n\n- Determine if flag usage should be less restrictive:\n  - Enable specifying multiple times?\n  - Specifying the kustomization file itself?\n  - Specifying it along with `-f` (separately)?\n- If / when available, gather usage metrics of the `-k` flag in kubectl commands to evaluate adoption\n- Gather feedback on overall flag experience from users (issues, slack, outreach, etc)\n- Should we add in-kubectl documentation for kustomization format? - e.g. `kubectl kustomize --help` would \n  give information about the kustomization.yaml format\n\n### Implement\n\n- Figure out better error messaging w.r.t. errors from the apiserver on output files vs input files\n- Feedback from evaluation\n\n## Docs\n\n- update all kubectl documentation that recommends piping `kustomize | kubectl` to use `-k`\n- update kubectl docs on k8s.io that create configmaps and secrets from Resource Config to also show` kustomization.yaml`\n- update kubectl docs on k8s.io that use `-n` to set namespace for apply to also show `kustomization.yaml`\n- update imperative kubectl docs on k8s.io that set namespaces, labels, annotations to also show the declarative\n  approach using kustomize\n  \n- Update cobra (e.g. `--help`) examples for apply, delete, get, etc to include the `-k` flag.\n- Update cobra docs for `-n` flag with apply to suggest using a declarative kustomization.yaml instead\n- Update cobra examples for imperative set, create commands that can be generated to call out the declarative\n  approaches.\n\n## Test plan\n\nThe following should be tests written for kubectl.\n\n- unit test to validate that the `-k` flag correctly invokes the kustomization library\n  - should invoke the library\n- unit test to validate that the `--kustomize` flag works the same as the `-k` flag\n  - should invoke the library\n- test to validate that the `-k` flag results in the kustomized resource config being provided to the commands\n  - should provide the expanded files\n- test to validate the `-k` flag there are resource files in the same directory\n  - should only pick up the kustomization, not other files\n- test to validate what happens if the `-k` flag is provided multiple times\n  - should fail\n- test to validate the `-k` flag if no kustomization file is present, but there are resource files\n  - should fail\n- test to validate the `-k` flag points to a kustomization.yaml\n  - should fail - directories only\n- test to validate the `-k` flag if `-f` or `--filename` is also provided\n  - should fail\n- test to validate the `-k` flag if it points directory with a file containing a kustomization resource\n  (group version kind), but not named `kustomization.yaml`\n  - should fail - kustomization.yaml only\n- test to validate that the `-k` flag can be opt-out in the cli-runtime.\n  - flag should not be registered if opt-out\n- TODO: add more tests here\n\n### Version Skew Tests\n\nNA.  This flag is a client only feature.\n\n## Implementation History\n\n## Alternatives\n"
+  },
+  {
+    "id": "8d14594a591549a8b73789495b81efed",
+    "title": "Kustomize Secret Generator Plugins",
+    "authors": ["@sethpollack"],
+    "owningSig": "sig-cli",
+    "participatingSigs": ["sig-apps", "sig-architecture"],
+    "reviewers": ["@monopole", "@Liujingfang1"],
+    "approvers": ["@monopole", "@Liujingfang1", "@pwittrock"],
+    "editor": "@sethpollack",
+    "creationDate": "2019-02-04",
+    "lastUpdated": "2019-02-04",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n[execRemoval]: https://github.com/kubernetes-sigs/kustomize/issues/692\n\n# Kustomize Secret K:V Generator Plugins\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n- [Risks and Mitigations](#risks-and-mitigations)\n  - [Several people want an \u003cem\u003eexec-style\u003c/em\u003e plugin](#several-people-want-an-exec-style-plugin)\n      - [mitigation](#mitigation)\n  - [goplugin limitations](#goplugin-limitations)\n    - [Not shareable as object code](#not-shareable-as-object-code)\n      - [mitigation](#mitigation-1)\n      - [No current support for Windows](#no-current-support-for-windows)\n      - [mitigation](#mitigation-2)\n    - [General symbol execution from the plugin](#general-symbol-execution-from-the-plugin)\n      - [mitigation](#mitigation-3)\n  - [Two means to specify legacy KV generation](#two-means-to-specify-legacy-kv-generation)\n      - [mitigation](#mitigation-4)\n- [Graduation Criteria of plugin framework](#graduation-criteria-of-plugin-framework)\n  - [Alpha status](#alpha-status)\n  - [Graduation to beta](#graduation-to-beta)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nKustomize users want to generate Kubernetes Secret\nobjects from general key:value (KV) pairs where the\nvalue is supposed to be a secret.  Users want to\ngenerate these pairs through integration with secret\nmanagement tools (e.g. see comments on\n[692][execRemoval]).\n\nCurrently Kustomize only supports reading secret values\nfrom local files which raises security concerns about\nfile lifetime and access. Reading secret values from\nthe execution of arbitrary \"commands\" in a\nkustomization.yaml file introduces concerns in a world\nwhere kustomization files can be used directly from the\ninternet when a user runs `kustomize build`.  This\nproposal describes the syntax for a new key:value\ngenerator plugin framework supporting an arbitrary\nnumber of plugin types to generate key:value pairs.\n\n## Motivation\n\nNot having a way to integrate `SecretGenerator` with\nsecret management tools requires hacky, insecure\nworkarounds.\n\n### Goals\n\n- In the `GeneratorArgs` section of a kustomization\n  file, a user may specify a plugin type, and a\n  specific instance of that type, for generating\n  key:value pairs.\n\n- The specification will allow for any number of plugin\n  types, and any number of instances of those types.\n  \n- The first type supported will be\n  [goplugins](https://golang.org/pkg/plugin), to enable\n  kustomize source code contributors to add custom KV\n  generators without the need to maintain a kustomize\n  source code fork.\n  \n  Kustomize maintainers expect developers who use a\n  goplugin to understand that a kustomize binary and\n  any goplugins expected to work with it must be\n  compiled on the same machine with the same compiler\n  against the same set of (transitive) libraries.\n  _It's the developers responsibility to bundle the\n  binary and shared libraries together in a container\n  image for actual use._\n\n- Other kinds of plugins, e.g. an _execute this binary_\n  plugin, should be subsequently easy to add, and could\n  be appropriate for end user use (but would require\n  consideration in a KEP first).\n\n\n### Non-Goals\n\n- Kustomize will not handle key:value generation plugin\n  installation/management, or seek to build a plugin\n  \"ecosystem\" of key:value generators.\n\n\n## Proposal\n\nThe current API of `SecretGenerator` looks like this:\n\n```\nsecretGenerator:\n- name: app-tls\n  files:\n  - secret/tls.cert\n  - secret/tls.key\n  type: \"kubernetes.io/tls\"\n\n- name: env_file_secret\n  env: env.txt\n  type: Opaque\n\n- name: myJavaServerEnvVars\n  literals:\n  - JAVA_HOME=/opt/java/jdk\n  - JAVA_TOOL_OPTIONS=-agentlib:hprof\n```\n\nThe proposed API would look like this:\n\n```\nsecretGenerator:\n- name: app-tls\n  kvSources:\n  - pluginType: builtin  // builtin is the default value of pluginType\n    name: files\n    args:\n    - secret/tls.cert\n    - secret/tls.key\n- name: env_file_secret\n  kvSources:\n  - name: env  // this is a builtin\n    args:\n    - env.txt\n- name: myJavaServerEnvVars\n  kvSources:\n  - name: literals    // this is a builtin\n    args:\n    - JAVA_HOME=/opt/java/jdk\n    - JAVA_TOOL_OPTIONS=-agentlib:hprof\n- name: secretFromPlugins\n  kvSources:\n  - pluginType: go      // described by this KEP\n    name: myplugin\n    args:\n    - someArg\n    - someOtherArg\n  - pluginType: kubectl      // some future KEP can write this\n    name: someKubectlPlugin\n    args:\n    - anotherArg\n    - yetAnotherArg\n```\n\n\nThe `kvSources` specified with `pluginType: builtin` are\nreformulations of existing key:value generators currently\ninvoked by the existing `dataSources` specification.\n\nA `kvSource` with `pluginType: Go` and `name: myplugin`\nwould attempt to load the file\n\n```\n~/.config/kustomize/plugins/kvSources/myplugin.so\n```\n\nand access the loaded plugin via the interface\n\n```\ntype KvSourcePlugin interface {\n\tGet() []kv.Pair\n}\n```\n\nThis clearly describes how the plugin must be formulated.\n\n\nThe loading implementation would look something like this:\n\n```\nfunc keyValuesFromPlugins(ldr ifc.Loader, plugins []types.Plugin) ([]kv.Pair, error) {\n\tvar allKvs []kv.Pair\n\n\tfor _, plugin := range plugins {\n\t\tsecretFunc, err := findPlugin(plugin.Name)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tkvs, err := secretFunc(ldr.Root(), plugin.Args)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tallKvs = append(allKvs, kvs...)\n\t}\n\n\treturn allKvs, nil\n}\n\nfunc findPlugin(name string) (func(string, []string) ([]kv.Pair, error), error) {\n\tallPlugins, err := filepath.Glob(os.ExpandEnv(\"$HOME/.config/kustomization_plugins/*.so\"))\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error loading plugins\")\n\t}\n\n\tfor _, filename := range allPlugins {\n\t\tp, err := plugin.Open(filename)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tsymbol, err := p.Lookup(name)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\n\t\tif secretFunc, exists := symbol.(func(string, []string) ([]kv.Pair, error)); exists {\n\t\t\treturn secretFunc, nil\n\t\t}\n\t}\n\n\treturn nil, fmt.Errorf(\"plugin %s not found\", name)\n}\n```\n\nAn example plugin would look like this:\n\n```\npackage main\n\nimport (\n\t\"os\"\n\n\t\"sigs.k8s.io/kustomize/pkg/kv\"\n)\n\nfunc Env(root string, args []string) ([]kv.Pair, error) {\n\tvar kvs []kv.Pair\n\tfor _, arg := range args {\n\t\tkvs = append(kvs, kv.Pair{Key: arg, Value: os.Getenv(arg)})\n\t}\n\treturn kvs, nil\n}\n```\n\n\nA developer - not an end user - would compile the\ngoplugin and main program like this:\n\n```\ngo build -buildmode=plugin ~/.config/kustomize/plugins/kvSources/myplugin.so\ngo get sigs.k8s.io/kubernetes-sigs/kustomize\n```\n\nthen bake the build artifacts into a container image\nfor use by an end user or continuous delivery bot.\n\n\n## Risks and Mitigations\n\n### Several people want an _exec-style_ plugin\n\n_exec-style_ means execute arbitrary code from some file,\nwith key value pairs being captured from the `stdout`\nof a subprocess.\n\n##### mitigation\n\nMost the lines of code written to implement this KEP\naccommodate the notion of KV generation via a _generic_\nnotion of a plugin, and a generic interface, in the\nkustomization file and associated structs and code.\n\nThis code recharactizes the three existing KV\ngeneration methods as `pluginType: builtin`\n(_literals_, _env_ and _files_), introduces the new\n`pluginType: Go`, and leaves room for someone to easily\nadd, say `pluginType: kubectl` to look for a kubectl\nplugin, and `pluginType: whatever` to handles some\ncompletely new style, e.g. look for the plugin name as\nan executable in some hardcoded location.\n\nActual implementation of these other kinds of plugins\nare _out of scope for this KEP_.\n\nThe ability to write a goplugin very specifically\ntargets a kustomize contributor who understands their\nlimitations.  An exec-style plugin would enable a\nbroader set of people, this time including end users.\nA KEP proposing such plugins would need to consider a\ndifferent set of risks and maintenance requirements.\n\n### goplugin limitations\n\nThe first plugin mechanism will be goplugins, since by\ndesign they are trivial to make for a Go program (like\nkustomize).  However, they have limitations, and are\nto some extent an experimental feature of Go tooling.\n\n#### Not shareable as object code\n\nThe shared object (`.so`) files created via `go build\n-buildmode=plugin` cannot be reliably used by a loading\nprogram unless both the program and the plugin were\ncompiled with the same version of Go, with the same\ntransitive libs, on the same machine (see, e.g. [this\ngolang\nissue](https://github.com/golang/go/issues/18827)).\n\nTherefore, the Go developer who elects to write their\nkey:value generator as a goplugin is obligated to compile a\nnew kustomize binary as well as the plugin.\n\nFor this developer, this extra compilation step will be far\neasier than forking kustomize and modifying the code\ndirectly to introduce all the piping one would need to\nexpress a new key:value secret generation style in a\nkustomization file and integrate it with existing methods.\n\nThe risk to not compiling, and, say using an `.so` that was\ncompiled long ago and far away is that the program will\npanic when the `build` command is executed.\n\n##### mitigation\n\nAn attempt to `kustomize build $target` that includes a\nkustomization file specifying a `pluginType: Go` will\nfail with an error message explaining the panic risk\nand demanding the additional use of the command line\nflag:\n\n```\nkustomize build --alpha_enable_goplugin_and_accept_panic_risk $target\n```\n\nThis flag signals that the feature 1) is alpha (it\ncould be removed), and 2) the feature could panic if\nimproperly used.\n\nAs noted above, developers who elect to use kustomize\nplus goplugins should bundle all compilation artifacts\ninto a container image for end use.  A bot using said\nimage would always specify the flag.\n\n##### No current support for Windows\n\nThis is mentioned in the\n[plugin package overview section](https://golang.org/pkg/plugin).\n\n##### mitigation\n\nThe target use case for secret generation is gitops\nrunning in a container on any kubernetes cluster.\n\nDevelopers on their workstations are not a target use\ncase, as they are less likely to have the security\nrequirements that necessitate using a plugin based\ngenerator.\n\nIf developers need this functionality, we need to\nbetter understand why. it is possible there is a better\nway of addressing their need. Of course, Windows users\nwho want to support goplugins could contribute to the\ngolang project.\n\n#### General symbol execution from the plugin\n\n##### mitigation\n\nA Go plugin, when loaded,\nwill be cast to a particular hardcoded interface, e.g.\n\n```\ntype KvSourcePlugin interface {\n\tGet() []kv.Pair\n}\n```\nand accessed exclusively through that interface.\n\n### Two means to specify legacy KV generation\n\nThe existing three mechanisms and syntax for generating KV\npairs (_literals_, _env_ and _files_) remain active, but\nadditionally the work for this KEP will allow these\nmechanisms to be expressed as `builtin` plugins (see example\nabove).\n\n##### mitigation\n\nIf plugins - both goplugins and other styles - prove\nunpopular or problematic, we can remove them per API change\npolicies.\n\nIf they do prove popular/useful, we can deprecate the legacy\nform for (_literals_, _env_ and _files_), and help people\nconvert to the new `builtin` form to access these KV sources.\n\n## Graduation Criteria of plugin framework\n\n### Alpha status \n\nThe kustomization fields that support a general plugin\nframework (which could support many kinds of plugins in\naddition to goplugins) will be implemented but\ndocumented as an alpha feature, which could be removed\nper API change policies. Loading of goplugins\nthemselves, as noted above, will be protected by a flag\ndenoting their alpha status.  As these features\ntarget contributors, this will be mentioned\nin CONTRIBUTING.md.\n\n### Graduation to beta\n\n* Exec-style plugins approaching GA.\n\n  As goplugins are targeted to kustomize contributors,\n  we'd like to see development of an exec-style plugin\n  targeted to end users before deciding to graduate\n  the framework to beta/GA.\n  \n  For goplugins themselves to reach beta/GA, we'd\n  like exec-style based plugin implemented and still\n  see some preference for the Go based approach.\n\n* Testing and documentation.\n\n  * High level feature test\n    (like those in [pkg/target](https://github.com/kubernetes-sigs/kustomize/tree/master/pkg/target))\n  * Field documentarion in the \n    [canonical example file](https://github.com/kubernetes-sigs/kustomize/blob/master/examples/helloWorld/kustomization.yaml)\n  * Usage [examples](https://github.com/kubernetes-sigs/kustomize/tree/master/examples).\n\n\n## Implementation History\n\n(TODO add PR's here)\n"
+  },
+  {
+    "id": "e77fa1f63eb9074105e4bcefed7a09d8",
+    "title": "Kustomize Subcommand Integration",
+    "authors": ["@Liujingfang1"],
+    "owningSig": "sig-cli",
+    "participatingSigs": ["sig-cli"],
+    "reviewers": ["@liggitt", "@seans3", "@soltysh"],
+    "approvers": ["@liggitt", "@seans3", "@soltysh"],
+    "editor": "@pwittrock",
+    "creationDate": "2018-11-07",
+    "lastUpdated": "2019-02-15",
+    "status": "implemented",
+    "seeAlso": [
+      "[kustomize](https://github.com/kubernetes-sigs/kustomize/blob/master/docs/workflows.md)",
+      "kustomize-file-processing-integration.md"
+    ],
+    "replaces": ["0008-kustomize.md"],
+    "supersededBy": ["n/a"],
+    "markdown": "\n# Kustomize Subcommand Integration\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n  - [Why this should be part of kubectl](#why-this-should-be-part-of-kubectl)\n- [Proposal](#proposal)\n  - [Justification for this approach](#justification-for-this-approach)\n  - [Justification for follow up](#justification-for-follow-up)\n- [Kustomize Example](#kustomize-example)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n  - [Copy kustomize code into staging](#copy-kustomize-code-into-staging)\n  - [Leave kustomize functionality separate from kubectl](#leave-kustomize-functionality-separate-from-kubectl)\n  - [Build a separate tools targeted at Kubernetes declarative workflows.](#build-a-separate-tools-targeted-at-kubernetes-declarative-workflows)\n\u003c!-- /toc --\u003e\n\n## Summary\n\n[Kustomize](https://github.com/kubernetes-sigs/kustomize)\nwas developed as a subproject of sig-cli by kubectl maintainers to address\na collection of [issues](#motivation)) creating friction for declarative workflows in kubectl\n(e.g. `kubectl apply`).  The\ngoal of the kustomize subproject was to bring this functionality back to kubectl to better complement\n`kubectl apply` and other declarative workflow commands.\n\n- declaratively generating Resource config\n- declaratively transforming Resource config\n- composing collections of Resource config across files and directories\n- layering the above on top of one another\n\nIt is independent of, but complementary to, the [*server-side apply*](https://github.com/kubernetes/enhancements/issues/555)\ninitiative that was started later and targeted at a separate collection of\n`kubectl apply` issues.\n\nKustomize offers generators and transformations in a declarative form that \nimprove on functionality provided by existing imperative commands in kubectl.\n\nThe declarative approach offers a clear path to accountability (all input can\nbe kept in version control), can safely exploit a holistic, unbounded view of\ndisparate resources and their interdependence (it's a plan about what to do, \nnot a direct action), and can be easily constrained to verifiable rules \nacross this view (all edits must be structured, no removal semantics, no \nenvironment side-effects, etc.).\n\nImperative kubectl commands / flags available through kustomize:\n\n- `kubectl create configmap`\n- `kubectl create secret`\n- `kubectl annotate`\n- `kubectl label`\n- `kubectl patch`\n- `-n` (namespace)\n- `-f \u003cfilename\u003e` (kubectl processes files with lists of Resources)\n\nKubectl commands / flags similar to what is available through kustomize:\n\n- `-f \u003cdir\u003e -R` (kubectl - recursing through directories, kustomize may follow references)\n- `kubectl set image` (kustomize directive to set the image tag only, not the image)\n\nThings in kustomize that are not imperative kubectl commands / flags:\n\n- `namePrefix` (prepend all resource names with this)\n- `nameSuffix` (append all resource names with this)\n- for a limited set of fields allow one field value to be set to match another\n\n## Motivation\n\n**Background:** Kubectl Apply\n\nKubectl apply is intended to provide a declarative workflow for working with the Kubernetes API.  Similar to kustomize,\n`apply` (client-side) pre-processes Resource Config and transforms it into Kubernetes patches sent to the\napiserver (transformation is a function of live cluster state and Resource Config).  Apply addresses user friction\nsuch as:\n\n- updating Resources from Resource Config without wiping out control-plane defined fields (e.g. Service clusterIp)\n- automatically deciding whether to create, update or delete Resources\n\n**Present:**\n\nHowever apply does contain user friction in its declarative workflow, the majority of which could be either reduced\nor solved through augmenting and leveraging capabilities already present in kubectl imperative commands from a\ndeclarative context.  To this end, kustomize was developed.\n\nExample GitHub issues addressed by kustomize:\n\n- Declarative Updates of Secrets + ConfigMaps\n [kubernetes/kubernetes#24744](https://github.com/kubernetes/kubernetes/issues/24744)\n- Declarative Updates of ConfigMaps (duplicate)\n [kubernetes/kubernetes#30337](https://github.com/kubernetes/kubernetes/issues/30337)\n- Collecting Resource Config Across multiple targets (e.g. files / urls)\n [kubernetes/kubernetes#24649](https://github.com/kubernetes/kubernetes/issues/24649)\n- Facilitate Rollouts of ConfigMaps on update\n [kubernetes/kubernetes#22368](https://github.com/kubernetes/kubernetes/issues/22368)\n- Transformation (and propagation) of Names, Labels, Selectors\n [kubernetes/kubernetes#1698](https://github.com/kubernetes/kubernetes/issues/1698)\n\nSome of the solutions provided by kustomize could also be done as bash scripts to generate and transform\nResource config using either kubectl or other commands (e.g. creating a Secret from a file).\n\nAs an example: [this](https://github.com/osixia/docker-openldap/tree/stable/example/kubernetes/using-secrets/environment)\nis a repo publishing Kubernetes Resource config and with it provides a bash script to help facilitate users generating\nSecret data from files.  e.g. We want to discourage as a pattern teaching users to run arbitrary bash scripts from\nGitHub.\n\nAnother example: [this](https://github.com/kubernetes/kubernetes/issues/23233) is suggesting writing a script to\ninvoke kubectl's patch and apply logic from a script (add-on manager circa 2016).\n\nUsers have asked for a declarative script-free way to apply changes to Resource Config.\n\n### Goals\n\nSolve common kubectl user friction (such as those defined in the Motivation section) by publishing kustomize\nfunctionality from kubectl to complement commands which are targeted at declarative workflows.\n\n- Complement commands targeted at declarative workflows: `kubectl apply`, `kubectl diff`\n- Complement the *server-side apply* initiative targeted at improving declarative workflows.\n\nUser friction solved through capabilities such as:\n\n- Generating Resource Config for Resources with data frequently populated from other sources - ConfigMap and Secret\n- Performing common cross-cutting transformations intended to be applied across Resource Configs - e.g.\n  name prefixing / suffixing, labeling, annotating, namespacing, imageTag setting.\n- Performing flexible targeted transformations intended to be applied to specific Resource Configs - e.g.\n  strategic patch, json patch\n- Composing and layering Resource Config with schema-aware transformations\n- Facilitating rolling updates to Resources such as ConfigMaps and Secrets via Resource Config transformations\n- Facilitating creation of Resources that require the creation to be ordered - e.g. namespaces, crds\n\n### Non-Goals\n\n- Exposing all imperative kubectl commands as declarative options\n- Exposing all kubectl generators and schema-aware transformations\n- Providing simpler alternatives to the APIs for declaring Resource Config (e.g. a simple way to create deployments)\n- Providing a templating or general substitution mechanism (e.g. for generating Resource Config)\n\n### Why this should be part of kubectl\n\n- It was purposefully built to address user friction in kubectl declarative workflows. Leaving these issues unaddressed\n  in the command itself reduces the quality of the product.\n- The techniques it uses to address the issues are based on the existing kubectl imperative commands.  It is\n  consistent with the rest of kubectl.\n- Bridging the imperative and declarative workflows helps bring a more integrated feel and consistent story to\n  kubectl's command set.\n- Kustomize is already part of the Kubernetes project and owned by SIG CLI (the same sig that owns kubectl).\n  SIG CLI members have expertise in the kustomize codebase and are committed to maintaining the solution going forward.\n- Providing it as part of kubectl will ensure that it is available to users of kubectl apply and simplify the\n  getting started experience.\n\n## Proposal\n\nKustomize Standalone Sub Command\n\nPublish the `kustomize build` command as `kubectl kustomize`.  Update \ndocumentation to demonstrate using kustomize as `kubectl kustomize \u003cdir\u003e | kubectl apply -f -`.\n\n`kubectl kustomize` takes a single argument with is the location of a directory containing a file named `kustomization.yaml`\nand writes to stdout the kustomized Resource Config.\n\nIf the directory does not contain a `kustomization.yaml` file, it returns an \nerror.\n\nDefer deeper integration into ResourceBuilder (e.g. `kubectl apply -k \u003cdir\u003e`) as a follow up after discussing\nthe precise UX and technical tradeoffs.  (i.e. deeper integration is more delicate and can be done independently.)\n\n### Justification for this approach\n\nThe end goal is to have kustomize fully integrated into cli-runtime as part of the Resource processing\nlibraries (e.g. ResourceBuilder) and supported by all commands that take the `-f` flag.\n\nIntroducing it as a standalone subcommand first is a simple way of introducing kustomize functionality to kubectl,\nand will be desirable even after kustomize is integrated into cli-runtime.  The benefits of introducing it\nas a standalone command are:\n\n- Users can view the kustomize output (i.e. without invoking other commands in dry-run mode)\n  - Allows users to experiment with kustomizations\n  - Allows users to debug kustomizations\n  - It is easier to educate users about kustomizations when they can run it independently\n- The subcommand UX is simple and well defined\n- The technical integration is simple and well defined\n\n### Justification for follow up\n\n- It can't address user friction requiring deeper integration - e.g. produce meaningful line numbers in error\n  messages and exit codes.\n- Most commands would require the same input pipe - e.g. get, delete, etc. would all need pipes.  Direct integration\n  is cleaner than always piping everything.\n- We haven't trained user with this pattern\n - We don't currently follow this pattern by default for other commands such as `kubectl create deployment`- which requires\n   additional flags to output content to pipe `--dry-run -o yaml`\n - We don't do this for file, dir or url targets - e.g. we don't do `curl something | kubectl apply -f -`\n- When both subcommand and ResourceBuilder integration were demoed at sig-cli, the integrated UX was preferred\n- Kustomize is the way we are addressing issues with declarative workflows so we should make it as simple and easy\n  to use as raw Resource Config files.\n\n## Kustomize Example\n\nFollowing is an example of a `kustomization.yaml` file used by kustomize:\n\n```\napiVersion: v1beta1\nkind: Kustomization\nnamePrefix: alices-\n\ncommonAnnotations:\n  oncallPager: 800-555-1212\n\nconfigMapGenerator:\n- name: myJavaServerEnvVars\n  literals:  \n  - JAVA_HOME=/opt/java/jdk\n  - JAVA_TOOL_OPTIONS=-agentlib:hprof\n\nsecretGenerator:\n- name: app-sec\n  files:\n  - secret/password\n  - secret/username\n```\n\nThe result of running `kustomize build` on this sample kustomizaiton.yaml file is:\n\n```\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: alices-myJavaServerEnvVars-7bc9c27cmf\n  annotations:\n    oncallPager: 800-555-1212\ndata:\n  JAVA_HOME: /opt/java/jdk\n  JAVA_TOOL_OPTIONS: -agentlib:hprof\napiVersion: v1\nkind: Secret\nmetadata:\n  name: alices-app-sec-c7c5tbh526\n  annotations:\n    oncallPager: 800-555-1212\ntype: Opaque\ndata:\n  password: c2VjcmV0Cg==\n  username: YWRtaW4K\n```\n\n### Implementation Details/Notes/Constraints\n\nIn contrast to `kubectl apply`, which was developed directly in kubernetes/kubernetes and had minimal usage or\nreal world experience prior, kustomize was built as an outside of kubernetes/kubernetes in the\nkubernetes-sigs/kustomize repo.  After `kubectl apply` was published, many issues were uncovered in it that should\nhave been discovered earlier.  Implementing kustomize independently allowed more time for gathering feedback and\nidentifying issues.\n\nKustomize library code will be moved from its current repository (location) to the cli-runtime repository used by\nkubectl.\n\n### Risks and Mitigations\n\nLow:\n\n- Kustomize can be run against remote urls.  A user could run it on a URL containing malicious workflows.  However this\n would only generate the config, and the user would need to pipe it to apply for the workloads to be run.  This is also\n true for `kubectl apply -f \u003curl\u003e` and or `kubectl create deployment --image \u003cbad image\u003e`.\n\nLow:\n\n- Kustomize has other porcelain commands to facilitate common workflows.  This proposal does not include integrating\n  them into kubectl.  Users would need to download kustomize separate to get these benefits.\n  \nLow:\n\n- `kubectl kustomize \u003cdir\u003e` doesn't take a `-f` flag like the other commands.\n\n## Graduation Criteria\n\nThe API version for kustomize is defined in the `kustomization.yaml` file.  The KEP is targeted `v1beta1`.\n\nThe criteria for graduating from `v1beta1` for the kustomize sub-command should be determined as part of\nevaluating the success and maturity of kustomize as a command within kubectl.\n\nMetrics for success and adoption could include but are not limited to:\n\n- number of `kustomization.yaml` files seen on sources such as GitHub\n- complexity (required) of `kustomization.yaml` files seen on sources such as GitHub.\n- (if available) number of calls to `kubectl kustomize` being performed\n- adoption or integration of kustomize by other tools\n\nMetrics for maturity and stability could include but are not limited to:\n\n- number and severity of kustomize related bugs filed that are intended to be fixed\n- the frequency of API changes and additions\n- understanding of relative use and importance of kustomize features\n\n**Note:** Being integrated into ResourceBuilder is *not* considered graduation and *not* gated on GA.\n\n## Implementation History\n\nMost implementation will be in cli-runtime\n\n- [ ] vendor `kustomize/pkg` into kubernetes\n- [ ] copy `kustomize/k8sdeps` into cli-runtime\n  - Once cli-runtime is out of k/k, move the kustomize libraries there (but \n  not the commands)\n- [ ] Implement a function in cli-runtime to run kustomize build with input as fSys and/or path. \n   - execute kustomize build to get a list of resources\n   - write the output to io.Writer\n- [ ] Add a subcommand `kustomize` in kubectl. This command accepts one argument \u003cdir\u003e and write the output to stdout\n      kubectl kustomize \u003cdir\u003e\n- [ ] documentation:\n  - Write full doc for `kubectl kustomize`\n  - Update the examples in kubectl apply/delete to include the usage of kustomize\n  \n## Alternatives\n\nThe approaches in this section are considered, but rejected.\n\n### Copy kustomize code into staging\n\nDon't wait until kubectl libraries are moved out of staging before integrating, immediately copy kustomize code into\nkubernetes/staging and move to this as the source of truth.\n\n- Pros\n - It could immeidately live with the other kubectl code owned by sig-cli, instead of waiting until this\n   is moved out.\n- Cons\n - We are trying to move code **out** of kubernetes/kubernetes.  This is doing the opposite.\n - We are trying to move kubectl **out** of kubernetes/kubernetes.  This is doing the opposite.\n\n### Leave kustomize functionality separate from kubectl\n\n- Pros\n  - It is (marginally) less engineering work\n- Cons\n  - It leaves long standing issues in kubectl unaddressed within the tool itself.\n  - It does not support any deeper integrations - such as giving error messages with meaningful line numbers.\n  \n### Build a separate tools targeted at Kubernetes declarative workflows.\n\nCopy the declarative code from kubectl into a new tool.  Use this for declarative workflows.\n\nQuestions:\n\n- Do we deprecate / remove it from kubectl or have it in both places?\n- If in both places do we need to support both?  Can they diverge?\n- What needs to be updated?  Docs, Blogs, etc.\n\n- Pros\n  - It makes kubectl simpler\n- Cons\n  - Not clear how this helps users\n  - Does't address distribution problems\n  - User friction around duplication of functionality or remove of functionality\n"
+  },
+  {
+    "id": "13f94e32d8c3f8ac448cb03508709524",
+    "title": "Reporting Conformance Test Results to Testgrid",
+    "authors": ["@andrewsykim"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": ["sig-testing", "sig-release"],
+    "reviewers": ["TBD"],
+    "approvers": ["TBD"],
+    "editor": "TBD",
+    "creationDate": "2018-06-06",
+    "lastUpdated": "2018-11-16",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Reporting Conformance Test Results to Testgrid\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n    - [How to Run E2E Conformance Tests](#how-to-run-e2e-conformance-tests)\n    - [Sonobuoy](#sonobuoy)\n      - [Installing Sonobuoy](#installing-sonobuoy)\n      - [Running Conformance Tests with Sonobuoy](#running-conformance-tests-with-sonobuoy)\n    - [Kubetest](#kubetest)\n      - [Installing Kubetest](#installing-kubetest)\n      - [Running Conformance Test with kubetest](#running-conformance-test-with-kubetest)\n    - [How to Upload Conformance Test Results to Testgrid](#how-to-upload-conformance-test-results-to-testgrid)\n      - [Requesting a GCS Bucket](#requesting-a-gcs-bucket)\n      - [Authenticating to your Testgrid Bucket](#authenticating-to-your-testgrid-bucket)\n      - [Uploading results to Testgrid](#uploading-results-to-testgrid)\n      - [Testgrid Configuration](#testgrid-configuration)\n    - [Lifecycle of Test Results](#lifecycle-of-test-results)\n    - [Examples](#examples)\n  - [Risks and Mitigations](#risks-and-mitigations)\n    - [Operational Overhead](#operational-overhead)\n    - [Misconfigured Tests](#misconfigured-tests)\n    - [Flaky Tests](#flaky-tests)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis is a KEP outlining the motivation behind why cloud providers should periodically upload E2E conformance test results to [Testgrid](https://github.com/kubernetes/test-infra/tree/master/testgrid) and how a cloud provider can go about doing this.\n\n## Motivation\n\nThe primary motivation behind collecting conformance test results from various cloud providers on a regular basis is to inform sig-release of any critical bugs. It's important to collect results from various cloud providers to increase coverage and inform other cloud providers of bugs they may be impacted by.\n\n### Goals\n\n* All SIGs / Subprojects owners representing a cloud provider is aware of the importance of frequently uploading conformance test results to testgrid\n* There is a clear and detailed process in place for any cloud provider to upload conformance test results to Testgrid.\n\n### Non-Goals\n\n* Test coverage - increasing test coverage is outside the scope of this KEP\n* CI/CD - what CI/CD tool is used to run E2E tests and upload results is outside the scope of this KEP. It is up to the cloud provider to decide where/how tests are run.\n* Cluster Provisioning - how a cluster is provisioned is outside the scope of this KEP.\n\n## Proposal\n\nWe would like to propose that every Kubernetes cloud provider reports conformance test results for every patch version of Kubernetes at the minimum. Running conformance tests against master and on pre-release versions are highly encouraged but will not be a requirement.\n\n### Implementation Details/Notes/Constraints\n\nBefore continuing, it is highly recommended you read the following documentation provided by sig-testing:\n* [Testgrid Configuration](https://github.com/kubernetes/test-infra/tree/master/testgrid#testgrid)\n* [Display Conformance Tests with Testgrid](https://github.com/kubernetes/test-infra/blob/master/testgrid/conformance/README.md)\n\n#### How to Run E2E Conformance Tests\n\nThis KEP outlines two ways of running conformance tests, the first using [Sonobuoy](https://github.com/heptio/sonobuoy) and the second using [kubetest](https://github.com/kubernetes/test-infra/tree/master/kubetest). Though Sonobuoy is easier to setup, it does not guarantee that you will run the latest set of conformance tests. Kubetest though requiring a bit more work to setup, can ensure that you are running the latest set of conformance tests.\n\nAt this point we will assume that you have a running cluster and that `kubectl` is configured to point to that cluster with admin access.\n\n#### Sonobuoy\n\nYou should use Sonobuoy if you would like to run the standard set of CNCF conformance tests. This may exclude any new tests added by the latest versions of Kubernetes.\n\n##### Installing Sonobuoy\n\nThe following is mostly a [copy of the sonobuoy documentation](https://github.com/heptio/sonobuoy#download-and-run).\n\nFirst install sonobuoy. The following command adds it to the `$GOBIN` environment variable which is expected to be part of your `$PATH` environment variable.\n```\n$ go get -u github.com/heptio/sonobuoy\n```\n\n##### Running Conformance Tests with Sonobuoy\n\nYou can then start e2e tests by simply running:\n```\n$ sonobuoy run\nRunning plugins: e2e, systemd-logs\nINFO[0001] created object                                name=heptio-sonobuoy namespace= resource=namespaces\nINFO[0001] created object                                name=sonobuoy-serviceaccount namespace=heptio-sonobuoy resource=serviceaccounts\nINFO[0001] created object                                name=sonobuoy-serviceaccount-heptio-sonobuoy namespace= resource=clusterrolebindings\nINFO[0001] created object                                name=sonobuoy-serviceaccount namespace= resource=clusterroles\nINFO[0001] created object                                name=sonobuoy-config-cm namespace=heptio-sonobuoy resource=configmaps\nINFO[0002] created object                                name=sonobuoy-plugins-cm namespace=heptio-sonobuoy resource=configmaps\nINFO[0002] created object                                name=sonobuoy namespace=heptio-sonobuoy resource=pods\nINFO[0002] created object                                name=sonobuoy-master namespace=heptio-sonobuoy resource=services\n```\n\nYou can then check the status of your e2e tests by running\n```\n$ sonobuoy status\nPLUGIN\t\tSTATUS\tCOUNT\ne2e\t\trunning\t1\nsystemd_logs\trunning\t3\n\nSonobuoy is still running. Runs can take up to 60 minutes.\n```\n\nE2E tests can take up to an hour. Once your tests are done, you can download a snapshot of your results like so:\n```\n$ sonobuoy retrieve .\n```\n\nOnce you have the following snapshot, extract it's contents like so\n```\n$ mkdir ./results; tar xzf *.tar.gz -C ./results\n```\n\nAt this point you should have the log file and JUnit results from your tests:\n```\nresults/plugins/e2e/results/e2e.log\nresults/plugins/e2e/results/junit_01.xml\n```\n\n#### Kubetest\n\nYou should use kubetest if you want to run the latest set of tests in upstream Kubernetes. This is highly recommended by sig-testing so that new tests can be accounted for in new releases.\n\n##### Installing Kubetest\n\nInstall kubetest using the following command which adds it to your `$GOBIN` environment variable which is expected to be part of your `$PATH` environment variable.\n```\ngo get -u k8s.io/test-infra/kubetest\n```\n\n##### Running Conformance Test with kubetest\n\nNow you can run conformance test with the following:\n```\ncd /path/to/k8s.io/kubernetes\nexport KUBERNETES_CONFORMANCE_TEST=y\nkubetest \\\n    # conformance tests aren't supposed to be aware of providers\n    --provider=skeleton \\\n    # tell ginkgo to only run conformance tests\n    --test --test_args=\"--ginkgo.focus=\\[Conformance\\]\" \\\n    # grab the most recent CI tarball of kubernetes 1.10, including the tests\n    --extract=ci/latest-1.10 \\\n    # directory to store junit results\n    --dump=$(pwd)/_artifacts | tee ./e2e.log\n```\n\nNote that `--extract=ci/latest-1.10` indicates that we want to use the binaries/tests on the latest version of 1.10. You can use `--extract=ci/latest` to run the latest set of conformance tests from master.\n\nOnce the tests have finished (takes about an hour) you should have the log file and JUnit results from your tests:\n```\ne2e.log\n_artifacts/junit_01.xml\n```\n\n#### How to Upload Conformance Test Results to Testgrid\n\n##### Requesting a GCS Bucket\n\nTestgrid requires that you store results in a publicly readable GCS bucket. If for whatever reason you cannot set up a GCS bucket, please contact @BenTheElder or more generally the [gke-kubernetes-engprod](mailto:gke-kubernetes-engprod@google.com) team to arrange for a Google [GKE](https://cloud.google.com/kubernetes-engine/) EngProd provided / maintained bucket for hosting your results.\n\n##### Authenticating to your Testgrid Bucket\n\nAssuming that you have a publicly readable bucket provided by the GKE team, you should have been provided a service account JSON file which you can use with [gcloud](https://cloud.google.com/sdk/downloads) to authenticate with your GCS bucket.\n\n```\n$ gcloud auth activate-service-account --key-file /path/to/k8s-conformance-serivce-accout.json\nActivated service account credentials for: [demo-bucket-upload@k8s-federated-conformance.iam.gserviceaccount.com]\n```\n\n##### Uploading results to Testgrid\n\nAt this point you should be able to upload your testgrid results to your GCS bucket. You can do so by running a python script availabile [here](https://github.com/kubernetes/test-infra/tree/master/testgrid/conformance). For this example, we upload results for v1.10 into it's own GCS prefix.\n```\ngit clone https://github.com/kubernetes/test-infra\ncd test-infra/testgrid/conformance\n./upload_e2e.py --junit /path/to/junit_01.xml \\\n       --log /path/to/e2e.log \\\n       --bucket=gs://k8s-conformance-demo/cloud-provider-demo/e2e-conformance-release-v1.10\nUploading entry to: gs://k8s-conformance-demo/cloud-provider-demo/e2e-conformance-release-v1.10/1528333637\nRun: ['gsutil', '-q', '-h', 'Content-Type:text/plain', 'cp', '-', 'gs://k8s-conformance-demo/cloud-provider-demo/e2e-conformance-release-v1.10/1528333637/started.json'] stdin={\"timestamp\": 1528333637}\nRun: ['gsutil', '-q', '-h', 'Content-Type:text/plain', 'cp', '-', 'gs://k8s-conformance-demo/cloud-provider-demo/e2e-conformance-release-v1.10/1528333637/finished.json'] stdin={\"timestamp\": 1528337316, \"result\": \"SUCCESS\"}\nRun: ['gsutil', '-q', '-h', 'Content-Type:text/plain', 'cp', '~/go/src/k8s.io/kubernetes/results/plugins/e2e/results/e2e.log', 'gs://k8s-conformance-demo/cloud-provider-demo/e2e-conformance-release-v1.10/1528333637/build-log.txt']\nRun: ['gsutil', '-q', '-h', 'Content-Type:text/plain', 'cp', '~/go/src/k8s.io/kubernetes/results/plugins/e2e/results/e2e.log', 'gs://k8s-conformance-demo/cloud-provider-demo/e2e-conformance-release-v1.10/1528333637/artifacts/e2e.log']\nDone.\n```\n\n##### Testgrid Configuration\n\nNext thing you want to do is configure testgrid to read results from your GCS bucket. There are two [configuration](https://github.com/kubernetes/test-infra/tree/master/testgrid#configuration) steps required. One for your [test group](https://github.com/kubernetes/test-infra/tree/master/testgrid#test-groups) and one for your [dashboard](https://github.com/kubernetes/test-infra/tree/master/testgrid#dashboards).\n\nTo add a test group update [config.yaml](https://github.com/kubernetes/test-infra/blob/master/config/testgrids/config.yaml) with something like the following:\n```\ntest_groups:\n...\n...\n- name: cloud-provider-demo-e2e-conformance-release-v1.10\n  gcs_prefix: k8s-conformance-demo/cloud-provider-demo/e2e-conformance-release-v1.10\n```\n\nTo add a link to your results in the testgrid dashboard, update [config.yaml](https://github.com/kubernetes/test-infra/blob/master/config/testgrids/config.yaml) with something like the following:\n```\ndashboards:\n...\n...\n- name: conformance-demo-cloud-provider\n  dashboard_tab:\n  - name: \"Demo Cloud Provider, v1.10\"\n    description: \"Runs conformance tests for cloud provier demo on release v1.10\"\n    test_group_name: cloud-provider-demo-e2e-conformance-release-v1.10\n```\n\nOnce you've made the following changes, open a PR against the test-infra repo adding the sig testing label (`/sig testing`) and cc'ing @kubernetes/sig-testing-pr-reviews. Once your PR merges you should be able to view your results on https://k8s-testgrid.appspot.com/ which should be ready to be consumed by the necessary stakeholders (sig-release, sig-testing, etc).\n\n#### Lifecycle of Test Results\n\nYou can configure the lifecycle of testgrid results by specifying fields like `days_of_results` on your test group configuration. More details about this in the [Testgrid Advanced Configuration](https://github.com/kubernetes/test-infra/tree/master/testgrid#advanced-configuration) docs. If for whatever reason you urgently need to delete testgrid results, you can contact someone from sig-testing.\n\n#### Examples\n\nHere are some more concrete examples of how other cloud providers are running conformance tests and uploading results to testgrid:\n* Open Stack\n    * [OpenLab zuul job for running/uploading testgrid results](https://github.com/theopenlab/openlab-zuul-jobs/tree/master/playbooks/cloud-provider-openstack-acceptance-test-e2e-conformance)\n    * [OpenStack testgrid config](https://github.com/kubernetes/test-infra/pull/7670)\n    * [OpenStack conformance tests dashboard](https://github.com/kubernetes/test-infra/pull/8154)\n\n\n### Risks and Mitigations\n\n#### Operational Overhead\n\nOperating CI/CD system to run conformance tests on a regular basis may incur extra work from every cloud provider. Though we anticipate the benefits of running conformance tests to outweight the operational overhead, in some cases it may not.\n\nMitigation: TODO\n\n#### Misconfigured Tests\n\nThere are various scenarios where cloud providers may mistakenly upload incorrect conformance tests results. One example being uploading results for the wrong Kubernetes version.\n\nMitigation: TODO\n\n#### Flaky Tests\n\nTests can fail for various reasons in any cloud environment and may raise false negatives for the release team.\n\nMitigation: TODO\n\n\n## Graduation Criteria\n\nAll providers are periodically uploading conformance test results in at least one of the methods outlined in this KEP.\n\n\n[umbrella issues]: TODO\n\n## Implementation History\n\n- Jun 6th 2018: KEP is merged as a signal of acceptance. Cloud providers should now be looking to report their conformance test results to testgrid.\n- Nov 19th 2018: KEP has been in implementation stage for roughly 5 months with Alibaba Cloud, Baidu Cloud, DigitalOcean, GCE, OpenStack and vSphere reporting conformance test results to testgrid.\n\n"
+  },
+  {
+    "id": "095872bb9efcbb267ae0d3ce0ec4ebac",
+    "title": "Cloud Controller Manager",
+    "authors": ["@cheftako", "@calebamiles", "@hogepodge"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["sig-cloud-provider", "sig-storage"],
+    "reviewers": ["@andrewsykim", "@calebamiles", "@hogepodge", "@jagosan"],
+    "approvers": ["@thockin"],
+    "editor": "TBD",
+    "creationDate": "2018-01-09",
+    "lastUpdated": "2019-04-10",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": ["contributors/design-proposals/cloud-provider/cloud-provider-refactoring.md"],
+    "supersededBy": null,
+    "markdown": "\n# Cloud Controller Manager\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Terms](#terms)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Intermediary Goals](#intermediary-goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Controller Manager Changes](#controller-manager-changes)\n  - [Kubelet Changes](#kubelet-changes)\n  - [API Server Changes](#api-server-changes)\n  - [Volume Management Changes](#volume-management-changes)\n  - [Deployment Changes](#deployment-changes)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n    - [Repository Requirements](#repository-requirements)\n      - [Notes for Repository Requirements](#notes-for-repository-requirements)\n      - [Repository Timeline](#repository-timeline)\n  - [Security Considerations](#security-considerations)\n- [Graduation Criteria](#graduation-criteria)\n  - [Graduation to Beta](#graduation-to-beta)\n    - [Process Goals](#process-goals)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Terms\n\n- **CCM**: Cloud Controller Manager - The controller manager responsible for running cloud provider dependent logic,\nsuch as the service and route controllers.\n- **KCM**: Kubernetes Controller Manager - The controller manager responsible for running generic Kubernetes logic,\nsuch as job and node_lifecycle controllers.\n- **KAS**: Kubernetes API Server - The core api server responsible for handling all API requests for the Kubernetes\ncontrol plane. This includes things like namespace, node, pod and job resources.\n- **K8s/K8s**: The core kubernetes github repository.\n- **K8s/cloud-provider**: Any or all of the repos for each cloud provider. Examples include [cloud-provider-gcp](https://github.com/kubernetes/cloud-provider-gcp),\n[cloud-provider-aws](https://github.com/kubernetes/cloud-provider-aws) and [cloud-provider-azure](https://github.com/kubernetes/cloud-provider-azure).\nWe have created these repos for each of the in-tree cloud providers. This document assumes in various places that the\ncloud providers will place the relevant code in these repos. Whether this is a long-term solution to which additional\ncloud providers will be added, or an incremental step toward moving out of the Kubernetes org is out of scope of this\ndocument, and merits discussion in a broader forum and input from SIG-Architecture and Steering Committee.\n- **K8s SIGs/library**: Any SIG owned repository.\n- **Staging**: Staging: Separate repositories which are currently visible under the K8s/K8s repo, which contain code\nconsidered to be safe to be vendored outside of the K8s/K8s repo and which should eventually be fully separated from\nthe K8s/K8s repo. Contents of Staging are prevented from depending on code in K8s/K8s which are not in Staging.\nControlled by [publishing kubernetes-rules-configmap](https://github.com/kubernetes/publishing-bot/blob/master/configs/kubernetes-rules-configmap.yaml)\n\n## Summary\n\nThis KEP outlines the architectural changes required to run the Cloud Controller Manager. This is part of a long-running effort\nto develop cloud provider specific features outside of Kubernetes core (kuberetes/kubernetes repo).\n\n## Motivation\n\nWe are trying to remove any dependencies from Kubernetes Core to any specific cloud provider. Currently we have seven\nsuch dependencies. To prevent this number from growing we have locked Kubernetes Core to the addition of any new\ndependencies. This means all new cloud providers have to implement all their pieces outside of the Core.\nHowever everyone still ends up consuming the current set of seven in repo dependencies. For the seven in repo cloud\nproviders any changes to their specific cloud provider code requires OSS PR approvals and a deployment to get those\nchanges in to an official build. The relevant dependencies require changes in the following areas.\n\n- [Kube Controller Manager](https://kubernetes.io/docs/reference/generated/kube-controller-manager/) - Track usages of [CMServer.CloudProvider](https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-controller-manager/app/options/options.go)\n- [API Server](https://kubernetes.io/docs/reference/generated/kube-apiserver/) - Track usages of [ServerRunOptions.CloudProvider](https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-apiserver/app/options/options.go)\n- [Kubelet](https://kubernetes.io/docs/reference/generated/kubelet/) - Track usages of [KubeletFlags.CloudProvider](https://github.com/kubernetes/kubernetes/blob/master/cmd/kubelet/app/options/options.go)\n- [How Cloud Provider Functionality is deployed to and enabled in the cluster](https://kubernetes.io/docs/setup/pick-right-solution/#hosted-solutions) - Track usage from [PROVIDER_UTILS](https://github.com/kubernetes/kubernetes/blob/master/cluster/kube-util.sh)\n\nFor the cloud providers who are in repo (a.k.a in-tree), moving out would allow them to more quickly iterate on their solution and\ndecouple cloud provider fixes from open source releases. Moving the cloud provider code out of the in-tree\nprocesses means that these processes do not need to load/run unnecessary code for the environment they are in.\nWe would like to abstract a core controller manager library to help standardize the behavior of the cloud\ncontroller managers produced by each cloud provider. We would like to minimize the number and scope of controllers\nrunning in the cloud controller manager so as to minimize the surface area for per cloud provider deviation.\n\n### Goals\n\n- Have a generic controller manager library available for use by the per cloud provider controller managers.\n- Provide a mechanism to run out-of-tree provider that has feature parity to existing in-tree providers.\n\n### Intermediary Goals\n\nHave a cloud controller manager in the kubernetes main repo which hosts all of\nthe controller loops for the in repo cloud providers.\nDo not run any cloud provider logic in the kube controller manager, the kube apiserver or the kubelet.\nAt intermediary points we may just move some of the cloud specific controllers out. (Eg. volumes may be later than the rest)\n\n### Non-Goals\n\n- Removing and fully deprecating in-tree cloud provider code. That will be coverd in KEP-0013.\n\n## Proposal\n\n### Controller Manager Changes\n\nFor the controller manager we would like to create a set of common code which can be used by both the cloud controller\nmanager and the kube controller manager. The cloud controller manager would then be responsible for running controllers\nwhose function is specific to cloud provider functionality. The kube controller manager would then be responsible\nfor running all controllers whose function was not related to a cloud provider.\n\nIn order to create a 100% cloud independent controller manager, the controller-manager will be split into multiple binaries.\n\n1. Cloud dependent controller-manager binaries\n2. Cloud independent controller-manager binaries - This is the existing `kube-controller-manager` that is being shipped\nwith kubernetes releases.\n\nThe cloud dependent binaries will run those loops that rely on cloudprovider in a separate process(es) within the kubernetes control plane.\nThe rest of the controllers will be run in the cloud independent controller manager.\nThe decision to run entire controller loops, rather than only the very minute parts that rely on cloud provider was made\nbecause it makes the implementation simple. Otherwise, the shared data structures and utility functions have to be\ndisentangled, and carefully separated to avoid any concurrency issues. This approach among other things, prevents code\nduplication and improves development velocity.\n\nNote that the controller loop implementation will continue to reside in the core repository. It takes in\ncloudprovider.Interface as an input in its constructor. Vendor maintained cloud-controller-manager binary could link\nthese controllers in, as it serves as a reference form of the controller implementation.\n\nThere are four controllers that rely on cloud provider specific code. These are node controller, service controller,\nroute controller and attach detach controller. Copies of each of these controllers have been bundled together into\none binary. The cloud dependent binary registers itself as a controller, and runs the cloud specific controller loops\nwith the user-agent named \"external-controller-manager\".\n\nRouteController and serviceController are entirely cloud specific. Therefore, it is really simple to move these two\ncontroller loops out of the cloud-independent binary and into the cloud dependent binary.\n\nNodeController does a lot more than just talk to the cloud. It does the following operations -\n\n1. CIDR management\n2. Monitor Node Status\n3. Node Pod Eviction\n\nWhile Monitoring Node status, if the status reported by kubelet is either 'ConditionUnknown' or 'ConditionFalse', then\nthe controller checks if the node has been deleted from the cloud provider. If it has already been deleted from the\ncloud provider, then it deletes the nodeobject without waiting for the `monitorGracePeriod` amount of time. This is the\nonly operation that needs to be moved into the cloud dependent controller manager.\n\nFinally, The attachDetachController is tricky, and it is not simple to disentangle it from the controller-manager\neasily, therefore, this will be addressed with Flex Volumes (Discussed under a separate section below)\n\n\nThe kube-controller-manager has many controller loops. [See NewControllerInitializers](https://github.com/kubernetes/kubernetes/blob/release-1.9/cmd/kube-controller-manager/app/controllermanager.go#L332)\n\n - [nodeIpamController](https://github.com/kubernetes/kubernetes/tree/release-1.10/pkg/controller/nodeipam)\n - [nodeLifecycleController](https://github.com/kubernetes/kubernetes/tree/release-1.10/pkg/controller/nodelifecycle)\n - [volumeController](https://github.com/kubernetes/kubernetes/tree/release-1.9/pkg/controller/volume)\n - [routeController](https://github.com/kubernetes/kubernetes/tree/release-1.9/pkg/controller/route)\n - [serviceController](https://github.com/kubernetes/kubernetes/tree/release-1.9/pkg/controller/service)\n - replicationController\n - endpointController\n - resourceQuotaController\n - namespaceController\n - deploymentController\n - etc..\n\nAmong these controller loops, the following are cloud provider dependent.\n\n - [nodeIpamController](https://github.com/kubernetes/kubernetes/tree/release-1.10/pkg/controller/nodeipam)\n - [nodeLifecycleController](https://github.com/kubernetes/kubernetes/tree/release-1.10/pkg/controller/nodelifecycle)\n - [volumeController](https://github.com/kubernetes/kubernetes/tree/release-1.9/pkg/controller/volume)\n - [routeController](https://github.com/kubernetes/kubernetes/tree/release-1.9/pkg/controller/route)\n - [serviceController](https://github.com/kubernetes/kubernetes/tree/release-1.9/pkg/controller/service)\n\nThe nodeIpamController uses the cloudprovider to handle cloud specific CIDR assignment of a node. Currently the only\ncloud provider using this functionality is GCE. So the current plan is to break this functionality out of the common\nversion of the nodeIpamController. Most cloud providers can just run the default version of this controller. However any\ncloud provider which needs cloud specific version of this functionality and disable the default version running in the\nKCM and run their own version in the CCM.\n\nThe nodeLifecycleController uses the cloudprovider to check if a node has been deleted from/exists in the cloud.\nIf cloud provider reports a node as deleted, then this controller immediately deletes the node from kubernetes.\nThis check removes the need to wait for a specific amount of time to conclude that an inactive node is actually dead.\nThe current plan is to move this functionality into its own controller, allowing the nodeIpamController to remain in\nK8s/K8s and the Kube Controller Manager.\n\nThe volumeController uses the cloudprovider to create, delete, attach and detach volumes to nodes. For instance, the\nlogic for provisioning, attaching, and detaching a EBS volume resides in the AWS cloudprovider. The volumeController\nuses this code to perform its operations.\n\nThe routeController configures routes for hosts in the cloud provider.\n\nThe serviceController maintains a list of currently active nodes, and is responsible for creating and deleting\nLoadBalancers in the underlying cloud.\n\n### Kubelet Changes\n\nMoving on to the kubelet, the following cloud provider dependencies exist in kubelet.\n\n - Find the cloud nodename of the host that kubelet is running on for the following reasons :\n      1. To obtain the config map for the kubelet, if one already exists\n      2. To uniquely identify current node using nodeInformer\n      3. To instantiate a reference to the current node object\n - Find the InstanceID, ProviderID, ExternalID, Zone Info of the node object while initializing it\n - Periodically poll the cloud provider to figure out if the node has any new IP addresses associated with it\n - It sets a condition that makes the node unschedulable until cloud routes are configured.\n - It allows the cloud provider to post process DNS settings\n\nThe majority of the calls by the kubelet to the cloud is done during the initialization of the Node Object. The other\nuses are for configuring Routes (in case of GCE), scrubbing DNS, and periodically polling for IP addresses.\n\nAll of the above steps, except the Node initialization step can be moved into a controller. Specifically, IP address\npolling, and configuration of Routes can be moved into the cloud dependent controller manager.\n\n[Scrubbing DNS was found to be redundant](https://github.com/kubernetes/kubernetes/pull/36785). So, it can be disregarded. It is being removed.\n\nFinally, Node initialization needs to be addressed. This is the trickiest part. Pods will be scheduled even on\nuninitialized nodes. This can lead to scheduling pods on incompatible zones, and other weird errors. Therefore, an\napproach is needed where kubelet can create a Node, but mark it as \"NotReady\". Then, some asynchronous process can\nupdate it and mark it as ready. This is now possible because of the concept of Taints.\n\nThis approach requires kubelet to be started with known taints. This will make the node unschedulable until these\ntaints are removed. The external cloud controller manager will asynchronously update the node objects and remove the\ntaints.\n\n### API Server Changes\n\nFinally, in the kube-apiserver, the cloud provider is used for transferring SSH keys to all of the nodes, and within an\nadmission controller for setting labels on persistent volumes.\n\nKube-apiserver uses the cloud provider for two purposes\n\n1. Distribute SSH Keys - This can be moved to the cloud dependent controller manager\n2. Admission Controller for PV - This can be refactored using the taints approach used in Kubelet\n\n### Volume Management Changes\n\nVolumes need cloud providers, but they only need **specific** cloud providers. The majority of volume management logic\nresides in the controller manager. These controller loops need to be moved into the cloud-controller manager. The cloud\ncontroller manager also needs a mechanism to read parameters for initialization from cloud config. This can be done via\nconfig maps.\n\nThere are two entirely different approach to refactoring volumes -\n[Flex Volumes](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-storage/flexvolume.md) and\n[CSI Container Storage Interface](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/container-storage-interface.md). There is an undergoing effort to move all\nof the volume logic from the controller-manager into plugins called Flex Volumes. In the Flex volumes world, all of the\nvendor specific code will be packaged in a separate binary as a plugin. After discussing with @thockin, this was\ndecidedly the best approach to remove all cloud provider dependency for volumes out of kubernetes core. Some of the discovery\ninformation for this can be found at [https://goo.gl/CtzpVm](https://goo.gl/CtzpVm).\n\n### Deployment Changes\n\nThis change will introduce new binaries to the list of binaries required to run kubernetes. The change will be designed\nsuch that these binaries can be installed via `kubectl apply -f` and the appropriate instances of the binaries will be\nrunning.\n\nIssues such as monitoring, configuring the new binaries will generally be left to cloud provider. However they should\nensure that test runs upload the logs for these new processes to [test grid](https://k8s-testgrid.appspot.com/).\n\nApplying the cloud controller manager is the only step that is different in the upgrade process.\nIn order to complete the upgrade process, you need to apply the cloud-controller-manager deployment to the setup.\nA deployment descriptor file will be provided with this change. You need to apply this change using\n\n```\nkubectl apply -f cloud-controller-manager.yml\n```\n\nThis will start the cloud specific controller manager in your kubernetes setup.\n\nThe downgrade steps are also the same as before for all the components except the cloud-controller-manager.\nIn case of the cloud-controller-manager, the deployment should be deleted using\n\n```\nkubectl delete -f cloud-controller-manager.yml\n```\n\n### Implementation Details/Notes/Constraints\n\n#### Repository Requirements\n\n**This is a proposed structure, and may change during the 1.11 release cycle.\nWG-Cloud-Provider will work with individual sigs to refine these requirements\nto maintain consistency while meeting the technical needs of the provider\nmaintainers**\n\nEach cloud provider hosted within the `kubernetes` organization shall have a\nsingle repository named `kubernetes/cloud-provider-\u003cprovider_name\u003e`. Those\nrepositories shall have the following structure:\n\n* A `cloud-controller-manager` subdirectory that contains the implementation\n  of the provider-specific cloud controller.\n* A `docs` subdirectory.\n* A `docs/cloud-controller-manager.md` file that describes the options and\n  usage of the cloud controller manager code.\n* A `docs/testing.md` file that describes how the provider code is tested.\n* A `Makefile` with a `test` entrypoint to run the provider tests.\n\nAdditionally, the repository should have:\n\n* A `docs/getting-started.md` file that describes the installation and basic\n  operation of the cloud controller manager code.\n\nWhere the provider has additional capabilities, the repository may have\nthe following subdirectories that contain the common features:\n\n* `dns` for DNS provider code.\n* `cni` for the Container Network Interface (CNI) driver.\n* `csi` for the Container Storage Interface (CSI) driver.\n* `flex` for the Flex Volume driver.\n* `installer` for custom installer code.\n\nEach repository may have additional directories and files that are used for\nadditional feature that include but are not limited to:\n\n* Other provider specific testing.\n* Additional documentation, including examples and developer documentation.\n* Dependencies on provider-hosted or other external code.\n\n\n##### Notes for Repository Requirements\n\nThis purpose of these requirements is to define a common structure for the\ncloud provider repositories owned by current and future cloud provider SIGs.\nIn accordance with the [WG-Cloud-Provider Charter](https://docs.google.com/document/d/1m4Kvnh_u_9cENEE9n1ifYowQEFSgiHnbw43urGJMB64/edit#)\nto \"define a set of common expected behaviors across cloud providers\", this\nproposal defines the location and structure of commonly expected code.\n\nAs each provider can and will have additional features that go beyond expected\ncommon code, requirements only apply to the location of the\nfollowing code:\n\n* Cloud Controller Manager implementations.\n* Documentation.\n\nThis document may be amended with additional locations that relate to enabling\nconsistent upstream testing, independent storage drivers, and other code with\ncommon integration hooks may be added\n\nThe development of the\n[Cloud Controller Manager](https://github.com/kubernetes/kubernetes/tree/master/cmd/cloud-controller-manager)\nand\n[Cloud Provider Interface](https://github.com/kubernetes/kubernetes/blob/master/pkg/cloudprovider/cloud.go)\nhas enabled the provider SIGs to develop external providers that\ncapture the core functionality of the upstream providers. By defining the\nexpected locations and naming conventions of where the external provider code\nis, we will create a consistent experience for:\n\n* Users of the providers, who will have easily understandable conventions for\n  discovering and using all of the providers.\n* SIG-Docs, who will have a common hook for building or linking to externally\n  managed documentation\n* SIG-Testing, who will be able to use common entry points for enabling\n  provider-specific e2e testing.\n* Future cloud provider authors, who will have a common framework and examples\n  from which to build and share their code base.\n\n##### Repository Timeline\n\nTo facilitate community development, providers named in the\n[Makes SIGs responsible for implementations of `CloudProvider`](https://github.com/kubernetes/community/pull/1862)\npatch can immediately migrate their external provider work into their named\nrepositories.\n\nEach provider will work to implement the required structure during the\nKubernetes 1.11 development cycle, with conformance by the 1.11 release.\nWG-Cloud-Provider may actively change repository requirements during the\n1.11 release cycle to respond to collective SIG technical needs.\n\nAfter the 1.11 release all current and new provider implementations must\nconform with the requirements outlined in this document.\n\n### Security Considerations\n\nMake sure that you consider the impact of this feature from the point of view of Security.\n\n## Graduation Criteria\n\nHow will we know that this has succeeded?\nGathering user feedback is crucial for building high quality experiences and SIGs have the important responsibility of\nsetting milestones for stability and completeness.\nHopefully the content previously contained in [umbrella issues][] will be tracked in the `Graduation Criteria` section.\n\n[umbrella issues]: https://github.com/kubernetes/kubernetes/issues/42752\n\n### Graduation to Beta\n\nAs part of the graduation to `stable` or General Availability (GA), we have set\nboth process and technical goals.\n\n#### Process Goals\n\nWe propose the following repository structure for the cloud providers which\ncurrently live in `kubernetes/pkg/cloudprovider/providers/*`\n\n```\ngit@github.com:kubernetes/cloud-provider-wg\ngit@github.com:kubernetes/cloud-provider-aws\ngit@github.com:kubernetes/cloud-provider-azure\ngit@github.com:kubernetes/cloud-provider-gcp\ngit@github.com:kubernetes/cloud-provider-openstack\n```\n\nWe propose this structure in order to obtain\n\n- ease of contributor on boarding and off boarding by creating repositories under\n  the existing `kubernetes` GitHub organization\n- ease of automation turn up using existing tooling\n- unambiguous ownership of assets by the CNCF\n\nThe use of a tracking repository `git@github.com:kubernetes/wg-cloud-provider`\nis proposed to\n\n- create an index of all cloud providers which WG Cloud Provider believes\n  should be highlighted based on defined criteria for quality, usage, and other\n  requirements deemed necessary by the working group\n- serve as a location for tracking issues which affect all Cloud Providers\n- serve as a repository for user experience reports related to Cloud Providers\n  which live within the Kubernetes GitHub organization or desire to do so\n\nMajor milestones:\n\n- March 18, 2018: Accepted proposal for repository requirements.\n\n*Major milestones in the life cycle of a KEP should be tracked in `Implementation History`.\nMajor milestones might include\n\n- the `Summary` and `Motivation` sections being merged signaling SIG acceptance\n- the `Proposal` section being merged signaling agreement on a proposed design\n- the date implementation started\n- the first Kubernetes release where an initial version of the KEP was available\n- the version of Kubernetes where the KEP graduated to general availability\n- when the KEP was retired or superseded*\n\nThe ultimate intention of WG Cloud Provider is to prevent multiple classes\nof software purporting to be an implementation of the Cloud Provider interface\nfrom fracturing the Kubernetes Community while also ensuring that new Cloud\nProviders adhere to standards of quality and whose management follow Kubernetes\nCommunity norms.\n\n## Alternatives\n\nOne alternate to consider is the use of a side-car. The cloud-interface in tree could then be a [GRPC](https://github.com/grpc/grpc-go)\ncall out to that side-car. We could then leave the Kube API Server, Kube Controller Manager and Kubelet pretty much as is.\nWe would still need separate repos to hold the code for the side care and to handle cluster setup for the cloud provider.\nHowever we believe that different cloud providers will (already) want different control loops. As such we are likely to need\nsomething like the cloud controller manager anyway. From the perspective it seems easier to centralize the effort in that\ndirection. In addition it should limit the proliferation of new processes across the entire cluster.\n"
+  },
+  {
+    "id": "bd5d5b10e305f1c0c77474be0bce9f73",
+    "title": "Cloud Provider Documentation",
+    "authors": ["@d-nishi", "@hogepodge", "@andrewsykim"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": ["sig-docs", "sig-cluster-lifecycle"],
+    "reviewers": ["@andrewsykim", "@calebamiles", "@hogepodge", "@jagosan"],
+    "approvers": ["@andrewsykim", "@hogepodge", "@jagosan"],
+    "editor": "TBD",
+    "creationDate": "2018-07-31",
+    "lastUpdated": "2019-02-12",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n## Documentation Requirements for Kubernetes Cloud Providers\n\n### Table of Contents\n\n\u003c!-- toc --\u003e\n  - [Summary](#summary)\n  - [Motivation](#motivation)\n    - [Goals](#goals)\n    - [Non-Goals](#non-goals)\n  - [Proposal](#proposal)\n    - [Proposed Format for Documentation](#proposed-format-for-documentation)\n    - [Requirement 1: Example Manifests](#requirement-1-example-manifests)\n    - [Requirement 2: Resource Management](#requirement-2-resource-management)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n  - [Risks and Mitigations](#risks-and-mitigations)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Alternatives [optional]](#alternatives-optional)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n### Summary\nThis KEP describes the documentation requirements for both in-tree and out-of-tree cloud providers.\nThese requirements are meant to ensure the usage and integration of cloud providers with the rest of the Kubernetes ecosystem is concise and clear and that documentation layout across providers is consistent.\n\n### Motivation\nCurrently documentation for cloud providers for both in-tree and out-of-tree managers is limited in both scope, consistency, and quality. This KEP describes requirements to create and maintain consistent documentation across all cloud providers. By establishing these standards, SIG Cloud Provider will benefit the user-community by offering a single discoverable source of reliable documentation while relieving the SIG-Docs team from the burden of maintaining content from various cloud providers across many Kubernetes versions.\n\n#### Goals\n\n* Create a single source of truth that outlines what documentation from each provider should cover.\n* Ensure all Kubernetes cloud providers adhere to this doc format to provide a consistent experience for all users.\n* Ensure SIG Docs can confidently link to documentation by any Kubernetes cloud provider on any future releases.\n\n#### Non-Goals\n\n* Where in the Kubernetes website the cloud provider documentation will be hosted. This should be a decision made by SIG Docs based on the content given by each cloud provider.\n* Cloud provider documentation outside the scope of Kubernetes.\n\n### Proposal\n\n#### Proposed Format for Documentation\n\nThe following is a proposed format of how docs should be added to `k8s.io/cloud-provider-*` repositories.\n\n#### Requirement 1: Example Manifests\n\nProvide validated manifests for every component required for both in-tree and out-of-tree versions of your cloud provider. The contents of the manifest should contain a DaemonSet resource that runs these components on the control plane nodes or a systemd unit file. The goal of the example manifests is to provide enough details on how each component in the cluster should be configured. This should provide enough context for users to build their own manifests if desired.\n\n```\ncloud-provider-foobar/\n├── ...\n├── ...\n├── docs\n│   └── example-manifests\n│       └── in-tree/\n│           ├── apiserver.manifest                 # an example manifest of apiserver using the in-tree integation of this cloud provider\n│           ├── kube-controller-manager.manifest   # an example manifest of kube-controller-manager using the in-tree integration of this cloud provider\n│           ├── kubelet.manifest                   # an example manifest of kubelet using the in-tree integration of this cloud provider\n│       └── out-of-tree/\n│           ├── apiserver.manifest                 # an example manifest of apiserver using the out-of-tree integration of this cloud provider\n│           ├── kube-controller-manager.manifest   # an example manifest of kube-controller-manager using the out-of-tree integration of this cloud provider\n│           ├── cloud-controller-manager.manifest  # an example manifest of cloud-controller-manager using the out-of-tree integration of this cloud provider\n│           ├── kubelet.manifest                   # an example manifest of kubelet using out-of-tree integration of this cloud provider\n```\n\n#### Requirement 2: Resource Management\n\nList the latest annotations and labels that are cloud-provider dependent and can be used by the Kubernetes administrator. The contents of these files should be kept up-to-date as annotations/labels are deprecated/removed/added/updated. Labels and annotations should be grouped based on the resource they are applied to. For example the label `beta.kubernetes.io/instance-type` is applied to nodes so it should be added to `k8s.io/cloud-provider-foobar/docs/node/labels.md`.\n\n```\ncloud-provider-foobar/\n├── ...\n├── ...\n├── docs\n│   └── resources\n│       └── node/\n│           ├── labels.md        # outlines what annotations that can be used on a Node resource\n│           ├── annotations.md   # outlines what annotations that can be used on a Node resource\n│           ├── README.md        # outlines any other cloud provider specific details worth mentioning regarding Nodes\n│       └── service/\n│           ├── labels.md        # outlines what annotations that can be used on a Service resource\n│           ├── annotations.md   # outlines what annotations that can be used on a Service resource\n│           ├── README.md        # outlines any other cloud provider specific details worth mentioning regarding Services\n│       └── persistentvolumes/\n│           ├── ...\n│       └── ...\n│       └── ...\n```\n\n### Implementation Details/Notes/Constraints [optional]\n\nThe requirements above lists the bare minimum documentation that any cloud provider in the Kubernetes ecosystem should have. Cloud providers may choose to add more contents under the `docs/` directory as they see fit.\n\n### Risks and Mitigations\n\nThere are no risks and mitigation with respect to adding documentation for cloud providers. If there are any, they would already exist in the various places the cloud provider docs exists today and implementing this KEP would not increase those risks.\n\n### Graduation Criteria\n\n* All cloud providers have written docs that adhere to the format specified in this KEP.\n* SIG Docs is consuming docs written by each provider in a way that is easily consumable for Kubernetes' users.\n\n### Alternatives [optional]\n\n* SIG Docs can be solely responsible for writing documentation around integrating Kubernetes with the existing cloud providers. This alternative would not be efficient because it would require SIG Docs to understand the context/scope of any cloud provider work that happens upstream. Developers who work with cloud provider integrations are most fit to write the cloud provider integration docs.\n\n## Implementation History\n\n- July 31st 2018: KEP is merged as a signal of acceptance. Cloud providers should now be looking to add documentation for their provider according to this KEP.\n- Nov 19th 2018: KEP has been in implementation stage for roughly 4 months with Alibaba Cloud, Azure, DigitalOcean, OpenStack and vSphere having written documentation for their providers according to this KEP.\n- Feb 12th 2019: KEP has been updated to state the implementation details and goals more clearly\n\n"
+  },
+  {
+    "id": "33fc272b8dcb1200ed282b37d9731b5f",
+    "title": "Support Out-of-Tree AWS Cloud Provider",
+    "authors": ["@andrewsykim"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": null,
+    "reviewers": ["TBD"],
+    "approvers": ["TBD"],
+    "editor": "TBD",
+    "creationDate": "2019-01-25",
+    "lastUpdated": "2019-01-25",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Supporting Out-of-Tree AWS Cloud Provider\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [X] k/enhancements issue in release milestone and linked to KEP (https://github.com/kubernetes/enhancements/issues/631)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documentedbs\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nBuild support for the out-of-tree AWS cloud provider. This involves a well-tested version of the cloud-controller-manager \nthat has feature parity to the kube-controller-manager. \n\n## Motivation\n\nMotivation for supporting out-of-tree providers can be found in the [Cloud Controller Manager KEP](/keps/sig-cloud-provider/20180530-cloud-controller-manager.md). \nThis KEP is specifically tracking progress for the AWS cloud provider.\n\n### Goals\n\n* Develop/test/release the AWS cloud-controller-manager\n* Kubernetes clusters running on AWS should be running the cloud-controller-manager.\n\n### Non-Goals\n\n* Removing in-tree AWS cloud provider code, this effort falls under the [KEP for removing in-tree providers](https://github.com/kubernetes/enhancements/blob/master/keps/sig-cloud-provider/2019-01-25-removing-in-tree-providers.md).\n\n## Proposal\n\n### Implementation Details/Notes/Constraints [optional]\n\nTODO for SIG-AWS\n\n### Risks and Mitigations\n\nTODO for SIG-AWS\n\n## Design Details\n\n### Test Plan\n\nTODO for SIG-AWS\n\n### Graduation Criteria\n\nTODO for SIG-AWS\n\n### Upgrade / Downgrade Strategy\n\nTODO for SIG-AWS\n\n### Version Skew Strategy\n\nTODO for SIG-AWS\n\n## Implementation History\n\nTODO for SIG-AWS\n"
+  },
+  {
+    "id": "99d76be7142a43bd9650c3c805c1510b",
+    "title": "Support Out-of-Tree GCE Cloud Provider",
+    "authors": ["@andrewsykim"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": null,
+    "reviewers": ["TBD"],
+    "approvers": ["TBD"],
+    "editor": "TBD",
+    "creationDate": "2019-01-25",
+    "lastUpdated": "2019-01-25",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Supporting Out-of-Tree GCE Cloud Provider\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [X] k/enhancements issue in release milestone and linked to KEP (https://github.com/kubernetes/enhancements/issues/668)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documentedbs\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nBuild support for the out-of-tree GCE cloud provider. This involves a well-tested version of the cloud-controller-manager \nthat has feature parity to the kube-controller-manager. \n\n## Motivation\n\nMotivation for supporting out-of-tree providers can be found in the [Cloud Controller Manager KEP](/keps/sig-cloud-provider/20180530-cloud-controller-manager.md). \nThis KEP is specifically tracking progress for the GCE cloud provider.\n\n### Goals\n\n* Develop/test/release the GCE cloud-controller-manager\n* Kubernetes clusters running on GCE should be running the cloud-controller-manager.\n\n### Non-Goals\n\n* Removing in-tree GCE cloud provider code, this effort falls under the [KEP for removing in-tree providers](https://github.com/kubernetes/enhancements/blob/master/keps/sig-cloud-provider/2019-01-25-removing-in-tree-providers.md).\n\n## Proposal\n\n### Implementation Details/Notes/Constraints [optional]\n\nTODO for SIG-GCP\n\n### Risks and Mitigations\n\nTODO for SIG-GCP\n\n## Design Details\n\n### Test Plan\n\nTODO for SIG-GCP\n\n### Graduation Criteria\n\nTODO for SIG-GCP\n\n### Upgrade / Downgrade Strategy\n\nTODO for SIG-GCP\n\n### Version Skew Strategy\n\nTODO for SIG-GCP\n\n## Implementation History\n\nTODO for SIG-GCP\n"
+  },
+  {
+    "id": "ae5986ab2279a79699db66ca3c6fc5f9",
+    "title": "Support Out-of-Tree IBM Cloud Provider",
+    "authors": ["@andrewsykim"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": null,
+    "reviewers": ["TBD"],
+    "approvers": ["TBD"],
+    "editor": "TBD",
+    "creationDate": "2019-01-25",
+    "lastUpdated": "2019-01-25",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Supporting Out-of-Tree IBM Cloud Provider\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [X] k/enhancements issue in release milestone and linked to KEP (https://github.com/kubernetes/enhancements/issues/671)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documentedbs\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nBuild support for the out-of-tree IBM cloud provider. This involves a well-tested version of the cloud-controller-manager \nthat has feature parity to the kube-controller-manager. \n\n## Motivation\n\nMotivation for supporting out-of-tree providers can be found in the [Cloud Controller Manager KEP](/keps/sig-cloud-provider/20180530-cloud-controller-manager.md). \nThis KEP is specifically tracking progress for the IBM cloud provider.\n\n### Goals\n\n* Develop/test/release the IBM cloud-controller-manager\n* Kubernetes clusters running on IBM should be running the cloud-controller-manager.\n\n### Non-Goals\n\n* Removing in-tree IBM cloud provider code, this effort falls under the [KEP for removing in-tree providers](https://github.com/kubernetes/enhancements/blob/master/keps/sig-cloud-provider/2019-01-25-removing-in-tree-providers.md).\n\n## Proposal\n\n### Implementation Details/Notes/Constraints [optional]\n\nTODO for SIG-IBM\n\n### Risks and Mitigations\n\nTODO for SIG-IBM\n\n## Design Details\n\n### Test Plan\n\nTODO for SIG-IBM\n\n### Graduation Criteria\n\nTODO for SIG-IBM\n\n### Upgrade / Downgrade Strategy\n\nTODO for SIG-IBM\n\n### Version Skew Strategy\n\nTODO for SIG-IBM\n\n## Implementation History\n\nTODO for SIG-IBM\n"
+  },
+  {
+    "id": "893900f104ada59bed48bc69e1c707d8",
+    "title": "Support Out-of-Tree OpenStack Cloud Provider",
+    "authors": ["@andrewsykim", "@adisky"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": null,
+    "reviewers": ["@lingxiankong", "@chrigl", "@ramineni", "@kendallnelson"],
+    "approvers": ["TBD"],
+    "editor": "TBD",
+    "creationDate": "2019-01-25",
+    "lastUpdated": "2019-01-25",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Supporting Out-of-Tree OpenStack Cloud Provider\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [X] k/enhancements issue in release milestone and linked to KEP (https://github.com/kubernetes/enhancements/issues/669)\n- [X] KEP approvers have set the KEP status to `implementable`\n- [X] Design details are appropriately documentedbs\n- [X] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nBuild support for the out-of-tree OpenStack cloud provider. This involves a well-tested version of the cloud-controller-manager \nthat has feature parity to the kube-controller-manager. \n\n## Motivation\n\nMotivation for supporting out-of-tree providers can be found in the [Cloud Controller Manager KEP](/keps/sig-cloud-provider/20180530-cloud-controller-manager.md). \nThis KEP is specifically tracking progress for the OpenStack cloud provider.\n\n### Goals\n\n* Develop/test/release the OpenStack cloud-controller-manager\n* Kubernetes clusters running on OpenStack should be running the cloud-controller-manager.\n\n### Non-Goals\n\n* Removing in-tree OpenStack cloud provider code, this effort falls under the [KEP for removing in-tree providers](https://github.com/kubernetes/enhancements/blob/master/keps/sig-cloud-provider/2019-01-25-removing-in-tree-providers.md).\n\n## Proposal\nThe OpenStack Cloud Provider is implemented, tested and documented, It is being released with matching kubernetes version from [release v1.11](https://github.com/kubernetes/cloud-provider-openstack/releases). cloud-provider-openstack release 1.14, 1.15, 1.16 has been running in production.\n\n### Implementation Details/Notes/Constraints [optional]\nOpenStack Cloud Provider is implemented [here](https://github.com/kubernetes/cloud-provider-openstack/releases). This repository also hosts other drivers like CSI, ingress-controller etc. Cloud Provider OpenStack is in feature parity with the intree version. Removal of intree cloud providers is dependent on [In-tree Storage Migration to CSI Plugin Migration](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/csi-migration.md). The migration work for intree OpenStack Volume driver to CSI Driver is in progress.\n\n### Risks and Mitigations\n\nNone as of now\n## Design Details\n\n### Test Plan\nOpenStack Cloud Provider is well tested, CI running at [OpenLab](https://github.com/theopenlab/openlab-zuul-jobs), results are reported to kubernetes test grid.\n\n### Graduation Criteria\n\nThis feature is complete, well tested, in parity with intree openstack provider. Documents needs to be updated as per sig-cloud-provider guideline. \n\n### Upgrade / Downgrade Strategy\n\nTODO for SIG-OpenStack\n \n### Version Skew Strategy\n\nTODO for SIG-OpenStack\n\n## Implementation History\n- Implementation and testing completed in 2018.\n- Matching kubernetes versions released from v1.11, latest version is v1.16.\n"
+  },
+  {
+    "id": "a05b765d4c70d495dc3627eb51cd1a1f",
+    "title": "Support Out-of-Tree vSphere Cloud Provider",
+    "authors": ["@frapposelli", "@andrewsykim"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": null,
+    "reviewers": ["@frapposelli", "@cantbewong", "@andrewsykim", "@dvonthenen"],
+    "approvers": ["@frapposelli", "@cantbewong", "@andrewsykim", "@dvonthenen"],
+    "editor": "TBD",
+    "creationDate": "2019-01-25",
+    "lastUpdated": "2019-01-25",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Supporting Out-of-Tree vSphere Cloud Provider\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [X] k/enhancements issue in release milestone and linked to KEP (https://github.com/kubernetes/enhancements/issues/670)\n- [X] KEP approvers have set the KEP status to `implementable`\n- [X] Design details are appropriately documented\n- [X] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [X] Graduation criteria is in place\n- [X] \"Implementation History\" section is up-to-date for milestone\n- [X] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nBuild support for the out-of-tree vSphere cloud provider. This involves a well-tested version of the cloud-controller-manager \nthat has feature parity to the kube-controller-manager.  This KEP captures mostly implemented work already completed in the\n[Cloud Provider vSphere repository](https://github.com/kubernetes/cloud-provider-vsphere).\n\n\n## Motivation\n\nMotivation for supporting out-of-tree providers can be found in the [Cloud Controller Manager KEP](/keps/sig-cloud-provider/20180530-cloud-controller-manager.md). \nThis KEP is specifically tracking progress for the vSphere cloud provider.\n\n### Goals\n\n* Develop/test/release the vSphere cloud-controller-manager\n* Kubernetes clusters running on vSphere should be running the cloud-controller-manager.\n\n### Non-Goals\n\n* Removing in-tree vSphere cloud provider code, this effort falls under the [KEP for removing in-tree providers](https://github.com/kubernetes/enhancements/blob/master/keps/sig-cloud-provider/2019-01-25-removing-in-tree-providers.md).\n\n## Proposal\n\nThe vSphere Cloud Provider is implemented, tested and partially documented. The next major steps are to migrate existing users of the in-tree provider to the external provider.\n\n### Implementation Details/Notes/Constraints [optional]\n\nMain provider work is completed and feature parity with in-tree will be achieved with the beta version. Removing in-tree `vsphere_volume` code is underway in the same repo and will piggyback on and be tracked through the\n[In-tree Storage Migration to CSI Plugin Migration](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/csi-migration.md)\nwork. New features will need to follow the community KEP process.\n\n### Risks and Mitigations\n\nNone known\n\n## Design Details\n\n### Test Plan\n\nThird-party testing of the vSphere Cloud Provider is handled by VMware and reports to testgrid.\n\n### Graduation Criteria\n\nThis feature is complete and will be ready for graduation with release 1.14.\n\n### Upgrade / Downgrade Strategy\n\nProjects that depend on the in-tree cloud provider can retain support for older versions of the provider that ships with Kubernetes for the time being. New deployment tooling is strongly encouraged to migrate to the external provider for future Kubernetes releases. A special consideration was also made for maintaining backward compatibility with in-tree vSphere cloud provider configuration file, an automation tool to upgrade from in-tree to out-of-tree provider was also created: [vcpctl](https://github.com/kubernetes/cloud-provider-vsphere/tree/master/cmd/vcpctl).\n\n### Version Skew Strategy\n\nAs such Cloud Provider vSphere has no version skew strategy for migrating from in-tree to out-of-tree providers. Future skew will use the facilities available to the Cloud Controller Manager interface.\n\n## Implementation History\n\n- Implementation and testing completed in 2018.\n- Feature parity with in-tree to be achieved with the 1.14 release.\n"
+  },
+  {
+    "id": "6a61803998328d6928e96dc972efef55",
+    "title": "Removing In-Tree Cloud Providers",
+    "authors": ["@andrewsykim", "@cheftako"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": ["sig-apps", "sig-api-machinery", "sig-network", "sig-storage"],
+    "reviewers": [
+      "@andrewsykim",
+      "@cheftako",
+      "@d-nishi",
+      "@dims",
+      "@hogepodge",
+      "@mcrute",
+      "@steward-yu"
+    ],
+    "approvers": ["@thockin", "@liggit"],
+    "editor": "TBD",
+    "creationDate": "2018-12-18",
+    "lastUpdated": "2019-04-11",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Removing In-Tree Cloud Provider Code\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Terms](#terms)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Approach](#approach)\n  - [Staging Directory](#staging-directory)\n    - [Cloud Provider Instances](#cloud-provider-instances)\n- [Alternatives](#alternatives)\n  - [Staging Alternatives](#staging-alternatives)\n    - [Git Filter-Branch](#git-filter-branch)\n  - [Build Location Alternatives](#build-location-alternatives)\n    - [Build K8s/K8s from within K8s/Cloud-provider](#build-k8sk8s-from-within-k8scloud-provider)\n    - [Build K8s/Cloud-provider within K8s/K8s](#build-k8scloud-provider-within-k8sk8s)\n  - [Config Alternatives](#config-alternatives)\n    - [Use component config to determine where controllers run](#use-component-config-to-determine-where-controllers-run)\n\u003c!-- /toc --\u003e\n\n## Terms\n\n- **CCM**: Cloud Controller Manager - The controller manager responsible for running cloud provider dependent logic,\nsuch as the service and route controllers.\n- **KCM**: Kubernetes Controller Manager - The controller manager responsible for running generic Kubernetes logic,\nsuch as job and node_lifecycle controllers.\n- **KAS**: Kubernetes API Server - The core api server responsible for handling all API requests for the Kubernetes\ncontrol plane. This includes things like namespace, node, pod and job resources.\n- **K8s/K8s**: The core kubernetes github repository.\n- **K8s/cloud-provider**: Any or all of the repos for each cloud provider. Examples include [cloud-provider-gcp](https://github.com/kubernetes/cloud-provider-gcp),\n[cloud-provider-aws](https://github.com/kubernetes/cloud-provider-aws) and [cloud-provider-azure](https://github.com/kubernetes/cloud-provider-azure).\nWe have created these repos for each of the in-tree cloud providers. This document assumes in various places that the\ncloud providers will place the relevant code in these repos. Whether this is a long-term solution to which additional\ncloud providers will be added, or an incremental step toward moving out of the Kubernetes org is out of scope of this\ndocument, and merits discussion in a broader forum and input from SIG-Architecture and Steering Committee.\n- **K8s SIGs/library**: Any SIG owned repository.\n- **Staging**: Staging: Separate repositories which are currently visible under the K8s/K8s repo, which contain code\nconsidered to be safe to be vendored outside of the K8s/K8s repo and which should eventually be fully separated from\nthe K8s/K8s repo. Contents of Staging are prevented from depending on code in K8s/K8s which are not in Staging.\nControlled by [publishing kubernetes-rules-configmap](https://github.com/kubernetes/publishing-bot/blob/master/configs/kubernetes-rules-configmap.yaml)\n- **In-tree**: code that lives in the core Kubernetes repository [k8s.io/kubernetes](https://github.com/kubernetes/kubernetes/).\n- **Out-of-Tree**: code that lives in an external repository outside of [k8s.io/kubernetes](https://github.com/kubernetes/kubernetes/).\n\n## Summary\n\nThis is a proposal outlining steps to remove \"in-tree\" cloud provider code from the k8s.io/kubernetes repo while being\nas least disruptive to end users and other Kubernetes developers as possible.\n\n## Motivation\n\nMotiviation behind this effort is to allow cloud providers to develop and make releases independent from the core\nKubernetes release cycle. The de-coupling of cloud provider code allows for separation of concern between \"Kubernetes core\"\nand the cloud providers within the ecosystem. In addition, this ensures all cloud providers in the ecosystem are integrating with\nKubernetes in a consistent and extendable way.\n\nHaving all cloud providers developed/released in their own external repos/modules will result in the following benefits:\n* The core pieces of Kubernetes (kubelet, kube-apiserver, kube-controller-manager, etc) will no longer depend on cloud provider specific\nAPIs (and their dependencies). This results in smaller binaries and lowers the chance of security vulnerabilities via external dependencies.\n* Each cloud provider is free to release features/bug fixes at their own schedule rather than relying on the core Kubernetes release cycle.\n\n### Goals\n\n- Remove all cloud provider specific code from the `k8s.io/kubernetes` repository with minimal disruption to end users and developers.\n\n### Non-Goals\n\n- Testing/validating out-of-tree provider code for all cloud providers, this should be done by each provider.\n\n## Proposal\n\n### Approach\n\nIn order to remove cloud provider code from `k8s.io/kubernetes`. A 3 phase approach will be taken.\n\n1. Move all code in `k8s.io/kubernetes/pkg/cloudprovider/providers/\u003cprovider\u003e` to `k8s.io/kubernetes/staging/src/k8s.io/legacy-cloud-providers/\u003cprovider\u003e/`. This requires removing all internal dependencies in each cloud provider to `k8s.io/kubernetes`.\n2. Begin to build/release the CCM from external repos (`k8s.io/cloud-provider-\u003cprovider\u003e`) with the option to import the legacy providers from `k8s.io/legacy-cloud-providers/\u003cprovider\u003e`. This allows the cloud-controller-manager to opt into legacy behavior in-tree (for compatibility reasons) or build new implementations of the provider. Development for cloud providers in-tree is still done in `k8s.io/kubernetes/staging/src/k8s.io/legacy-cloud-providers/\u003cprovider\u003e` during this phase.\n3. Delete all code in `k8s.io/kubernetes/staging/src/k8s.io/legacy-cloud-providers` and shift main development to `k8s.io/cloud-provider-\u003cprovider\u003e`. External cloud provider repos can optionally still import `k8s.io/legacy-cloud-providers` but it will no longer be imported from core components in `k8s.io/kubernetes`.\n\n#### Phase 1 - Moving Cloud Provider Code to Staging\n\nIn Phase 1, all cloud provider code in `k8s.io/kubernetes/pkg/cloudprovider/providers/\u003cprovider\u003e` will be moved to `k8s.io/kubernetes/staging/src/k8s.io/legacy-cloud-providers/\u003cprovider\u003e`. Reasons why we \"stage\" cloud providers as the first phase are:\n* The staged legacy provider repos can be imported from the out-of-tree provider if they choose to opt into the in-tree cloud provider implementation. This allows for a smoother transition between in-tree and out-of-tree providers in cases where there are version incompatibilites between the two.\n* Staging the cloud providers indicates to the community that they are slated for removal in the future.\n\nThe biggest challenge of this phase is to remove dependences to `k8s.io/kubernetes` in all the providers. This is a requirement of staging a repository and a best practice for consuming external dependencies. All other repos \"staged\" (`client-go`, `apimachinery`, `api`, etc) in Kubernetes follow the same pattern. The full list of internal dependencies that need to be removed can be found in issue [69585](https://github.com/kubernetes/kubernetes/issues/69585).\n\n#### Phase 2 - Building CCM from Provider Repos\n\nIn Phase 2, cloud providers will be expected to build the cloud controller manager from their respective provider repos (`k8s.io/cloud-provider-\u003cprovider\u003e`). Providers can choose to vendor in their legacy provider in `k8s.io/legacy-cloud-providers/\u003cprovider\u003e`, build implementations from scratch or both. Development in-tree is still done in the staging directories under the `k8s.io/kubernetes` repo.\n\nThe kube-controller-manager will still import the cloud provider implementations in staging. The package location of the provider implementations will change because each staged directory will be \"vendored\" in from their respective staging directory. The only change in core components is how the cloud providers are imported and the behavior of each cloud provider should not change.\n\n#### Phase 3 - Migrating Provider Code to Provider Repos\n\nIn Phase 3, all code in `k8s.io/kubernetes/staging/src/k8s.io/legacy-cloud-providers/\u003cprovider\u003e` will be removed and development of each cloud provider should be done in their respective external repos. It's important that by this phase, both in-tree and out-of-tree cloud providers are tested and production ready. Ideally most Kubernetes clusters in production should be using the out-of-tree provider before in-tree support is removed. A plan to migrate existing clusters from using the `kube-controller-manager` to the `cloud-controller-manager` is currently being developed. More details soon.\n\nExternal cloud providers can optionally still import providers from `k8s.io/legacy-cloud-providers` but no core components in `k8s.io/kubernetes` will import the legacy provider and the respective staging directory will be removed along with all its dependencies.\n\n### Staging Directory\n\nThere are several sections of code which need to be shared between the K8s/K8s repo and the K8s/Cloud-provider repos.\nThe plan for doing that sharing is to move the relevant code into the Staging directory as that is where we share code\ntoday. The current Staging repo has the following packages in it.\n- Api\n- Apiextensions-apiserver\n- Apimachinery\n- Apiserver\n- Client-go\n- Code-generator\n- Kube-aggregator\n- Metrics\n- Sample-apiserver\n- Sample-Controller\n\nWith the additions needed in the short term to make this work; the Staging area would now need to look as follows.\n- Api\n- Apiextensions-apiserver\n- Apimachinery\n- Apiserver\n- Client-go\n- **legacy-cloud-providers**\n- Code-generator\n- Kube-aggregator\n- Metrics\n- Sample-apiserver\n- Sample-Controller\n\n#### Cloud Provider Instances\n\nCurrently in K8s/K8s the cloud providers are actually included in the [providers.go](https://github.com/kubernetes/kubernetes/blob/master/pkg/cloudprovider/providers/providers.go)\nfile which then includes each of the in-tree cloud providers. In the short term, we would leave that file where it is\nand adjust it to point at the new homes under Staging. For the K8s/cloud-provider repo, would have the following CCM\nwrapper file. (Essentially a modified copy of cmd/cloud-controller-manager/controller-manager.go) The wrapper for each\ncloud provider would import just their vendored cloud-provider implementation rather than providers.go file.\n\nk8s/k8s: pkg/cloudprovider/providers/providers.go\n```package cloudprovider\n\nimport (\n  // Prior to cloud providers having been moved to Staging\n  _ \"k8s.io/kubernetes/pkg/cloudprovider/providers/aws\"\n  _ \"k8s.io/kubernetes/pkg/cloudprovider/providers/azure\"\n  _ \"k8s.io/kubernetes/pkg/cloudprovider/providers/cloudstack\"\n  _ \"k8s.io/kubernetes/pkg/cloudprovider/providers/gce\"\n  _ \"k8s.io/kubernetes/pkg/cloudprovider/providers/openstack\"\n  _ \"k8s.io/kubernetes/pkg/cloudprovider/providers/ovirt\"\n  _ \"k8s.io/kubernetes/pkg/cloudprovider/providers/photon\"\n  _ \"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere\"\n)\n```\n\nk8s/cloud-provider-gcp: pkg/cloudprovider/providers/providers.go\n```package cloudprovider\n\nimport (\n  // Cloud providers\n  _ \"k8s.io/legacy-cloud-providers/aws\"\n  _ \"k8s.io/legacy-cloud-providers/azure\"\n  _ \"k8s.io/legacy-cloud-providers/gce\"\n  _ \"k8s.io/legacy-cloud-providers/openstack\"\n  _ \"k8s.io/legacy-cloud-providers/vsphere\"\n)\n```\n\n## Alternatives\n\n### Staging Alternatives\n\n#### Git Filter-Branch\n\nOne possible alternative is to make use of a Git Filter Branch to extract a sub-directory into a virtual repo. The repo\nneeds to be sync'd in an ongoing basis with K8s/K8s as we want one source of truth until K8s/K8s does not pull in the\ncode. This has issues such as not giving K8s/K8s developers any indications of what the dependencies various\nK8s/Cloud-providers have. Without that information it becomes very easy to accidentally break various cloud providers\nand time you change dependencies in the K8s/K8s repo. With staging the dependency line is simple and [automatically\nenforced](https://github.com/kubernetes/kubernetes/blob/master/hack/verify-no-vendor-cycles.sh). Things in Staging are\nnot allowed to depend on things outside of Staging. If you want to add such a dependency you need to add the dependent\ncode to Staging. The act of doing this means that code should get synced and solve the problem. In addition the usage\nof a second different library and repo movement mechanism will make things more difficult for everyone.\n\n“Trying to share code through the git filter will not provide this protection. In addition it means that we now have\ntwo sharing code mechanisms which increases complexity on the community and build tooling. As such I think it is better\nto continue to use the Staging mechanisms. ”\n\n### Build Location Alternatives\n\n#### Build K8s/K8s from within K8s/Cloud-provider\n\nThe idea here is to avoid having to add a new build target to K8s/K8s. The K8s/Cloud-provider could have their own\ncustom targets for building things like KAS without other cloud-providers implementations linked in. It would also\nallow other customizations of the standard binaries to be created. While a powerful tool, this mechanism seems to\nencourage customization of these core binaries and as such to be discouraged. Providing the appropriate generic\nbinaries cuts down on the need to duplicate build logic for these core components and allow each optimization of build.\nDownload prebuilt images at a version and then just build the appropriate addons.\n\n#### Build K8s/Cloud-provider within K8s/K8s\n\nThe idea here would be to treat the various K8s/Cloud-provider repos as libraries. You would specify a build flavor and\nwe would pull in the relevant code based on what you specified. This would put tight restrictions on how the\nK8s/Cloud-provider repos would work as they would need to be consumed by the K8s/K8s build system. This seems less\nextensible and removes the nice loose coupling which the other systems have. It also makes it difficult for the cloud\nproviders to control their release cadence.\n\n### Config Alternatives\n\n#### Use component config to determine where controllers run\n\nCurrently KCM and CCM have their configuration passed in as command line flags. If their configuration were obtained\nfrom a configuration server (component config) then we could have a single source of truth about where each controller\nshould be run. This both solves the HA migration issue and other concerns about making sure that a controller only runs\nin 1 controller manager. Rather than having the controllers as on or off, controllers would now be configured to state\nwhere they should run, KCM, CCM, Nowhere, … If the KCM could handle this as a run-time change nothing would need to\nchange. Otherwise it becomes a slight variant of the proposed solution. This is probably the correct long term\nsolution. However for the timeline we are currently working with we should use the proposed solution.\n"
+  },
+  {
+    "id": "787f446f4d3557b3a43e62c1682b006c",
+    "title": "Promoting Cloud Provider Labels to GA",
+    "authors": ["@andrewsykim"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": ["sig-node", "sig-storage"],
+    "reviewers": ["@dims", "@liggit", "@msau42", "@saad-ali", "@thockin"],
+    "approvers": ["@thockin", "@liggit"],
+    "editor": "TBD",
+    "creationDate": "2019-02-15",
+    "lastUpdated": "2019-02-15",
+    "status": "implementable",
+    "seeAlso": ["/keps/sig-node/20190130-node-os-arch-labels.md"],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Promoting Cloud Provider Labels to GA\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Drawbacks [optional]](#drawbacks-optional)\n- [Alternatives [optional]](#alternatives-optional)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [X] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nWhen node and volume resources are created in Kubernetes, labels should be applied based on the underlying cloud provider of the Kubernetes cluster.\nThese labels contain cloud provider information that may be critical to some advanced features (mainly scheduling).\nWhen these labels were first introduced, they were prefixed with \"beta\" as the maturity and usage of these labels were not known at the time.\n\nToday, the cloud provider specific labels are:\n* `beta.kubernetes.io/instance-type`\n* `failure-domain.beta.kubernetes.io/zone`\n* `failure-domain.beta.kubernetes.io/region`\n\nThis KEP proposes to remove the beta labels and replace them with their GA equivalents:\n* `node.kubernetes.io/instance-type`\n* `topology.kubernetes.io/zone`\n* `topology.kubernetes.io/region`\n\n## Motivation\n\nThe labels mentioned above are consumed by almost all Kubernetes clusters that have cloud providers enabled. Given their maturity and widespread use, we should\npromote these labels from beta to GA.\n\n### Goals\n\n* promote cloud provider node/volume labels to GA with minimal visible changes to users.\n* remove the usage of \"beta\" cloud provider node/volume labels without breaking compatibility guaranetees. This will span many Kubernetes versions as per the Kubernetes deprecation policy.\n\n### Non-Goals\n\n* introducing more labels\n* changing the behaviour of these labels within the Kubernetes system.\n\n## Proposal\n\nIn order to promote these labels to GA safely, there will be a period in which both the \"beta\" and \"GA\" labels are applied to node and volume objects.\nThis is required in order to maintain backwards compatibility as many clusters rely on the beta labels today.\n\nFor the case of existing resources, keeping the beta labels is a requirement in order for existing workloads to behave as expected. A mechanism to populate existing resources\nwith the new GA versions of the labels will also be needed. For the case of new resources, both labels are still required as workloads may still consume the beta labels in some other resource\nthat was not updated yet to use the GA labels. One possible example is where a deployment may still use the beta zone label (`failure-domain.beta.kubernetes.io/zone`) as a\nnodeSelector and not applying the beta labels to new nodes would mean new nodes in that zone would not be considered when pods are being scheduled.\n\n### Implementation Details/Notes/Constraints [optional]\n\nHere is a break down of the implementation steps:\n\n1) [v1.17] update components to apply both the GA and beta labels to nodes \u0026 volumes.\n2) [v1.17] deprecate the beta labels.\n3) [v1.17] update the appropriate release notes \u0026 documentation to promote the use of GA labels over beta labels.\n4) [v1.18] continue to promote usage of GA labels over beta labels.\n5) [v1.19] continue to promote usage of GA labels over beta labels.\n6) [v1.20] continue to promote usage of GA labels over beta labels.\n7) [v1.21] components that consume the beta labels will be updated to only check for GA labels.\n8) [v1.21] stop applying beta labels to new resources, existing resources will continue to have those labels unless manually removed.\n\n### Risks and Mitigations\n\n* duplicate labels that do the same thing can be confusing/annoying for users\n* post v1.18 Kubernetes clusters may have danging labels that provide no function\n* improper handling of labels can lead to critical bugs in scheduling / volume topology / node registration / etc.\n\n## Design Details\n\n### Test Plan\n\n**Note:** *Section not required until targeted at a release.*\n\nTBD\n\n### Graduation Criteria\n\nLabels for zones, regions and instance-type have been beta since v1.3, they are widely adopted by Kubernetes users.\n\n\n### Upgrade / Downgrade Strategy\n\nThere is relatively low risk when it comes to upgrade / downgrade of clusters with respect to this enhancement.\nBecause we will apply both beta and GA labels to resources, a downgrade scenario would result in resources having a new label that may not necessarily be\nconsumed by anything else in the system yet. With the beta labels still in place, any features relying on these labels should continue to function as expected.\nWhen we stop applying beta labels to resources in v1.18, newly created resources will have the GA label _only_, but any existing resources carried over will have both\nthe GA labels and the beta labels. In this scearnio, a downgrade would only cause issues if a new node/volume resource was created\nin the newer version (v1.18 or greater) and other resources in the cluster still referenced the deprecated beta resource after a downgrade.\nThis edge case would only occur if users have not replaced usage of the beta labels with GA labels by v1.18.\n\n### Version Skew Strategy\n\nNo issues should arise from version skew assuming users do not replace usage of beta and GA labels until after all Kubernetes components are upgraded.\nIn the event that users attempt to update a workload to consume the GA labels in the middle of a cluster upgrade, workloads should eventually run as\nexpected once the upgrade is complete.\n\n## Implementation History\n\n- the `Summary` and `Motivation` sections being merged signaling SIG acceptance\n- the `Proposal` section being merged signaling agreement on a proposed design\n\n## Drawbacks [optional]\n\nThere are valid reasons why we should not move forward with this KEP. Replacing labels requires a lot of work to ensure plenty of time for deprecating warnings\nand that no existing behavior has changed. There is also a chance that users may choose (for whatever reason) to never replace beta labels with GA labels until something in the\nKubernetes cluster no longer works. This poses a risk to Kubernetes users that may indicate this effort is not worth the risk/time involved.\n\n## Alternatives [optional]\n\n* continue to use beta labels until a V2 of Nodes / PersistentVolumes is developed and breaking changes are acceptable.\n* continue to use existing beta labels forever\n"
+  },
+  {
+    "id": "d10f5bbc8119f16e30fa29e6c881aa0b",
+    "title": "Cloud Controller Manager Migration",
+    "authors": ["@andrewsykim"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": ["sig-api-machinery"],
+    "reviewers": ["@cheftako", "@nckturner"],
+    "approvers": ["@lavalamp"],
+    "editor": "TBD",
+    "creationDate": "2019-04-22",
+    "lastUpdated": "2019-04-22",
+    "status": "implementable",
+    "seeAlso": ["/keps/sig-cloud-provider/20180530-cloud-controller-manager.md"],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Cloud Controller Manager Migration\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n    - [Migration Configuration](#migration-configuration)\n    - [Component Flags](#component-flags)\n    - [Example Walkthrough of Controller Migration](#example-walkthrough-of-controller-migration)\n      - [Enable Leader Migration on Components](#enable-leader-migration-on-components)\n      - [Deploy the CCM](#deploy-the-ccm)\n      - [Update Leader Migration Config on Upgrade](#update-leader-migration-config-on-upgrade)\n      - [Disable Leader Migration](#disable-leader-migration)\n  - [Risks and Mitigations](#risks-and-mitigations)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n**ACTION REQUIRED:** In order to merge code into a release, there must be an issue in [kubernetes/enhancements] referencing this KEP and targeting a release milestone **before [Enhancement Freeze](https://github.com/kubernetes/sig-release/tree/master/releases)\nof the targeted release**.\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e., [kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be updated for the enhancement to be released.\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nSupport a migration process for large scale and highly available Kubernetes clusters using the in-tree cloud providers (via kube-controller-manager and kubelet) to their out-of-tree\nequivalents (via cloud-controller-manager).\n\n## Motivation\n\nSIG Cloud Provider is in the process of migrating the cloud specific code from the core Kubernetes tree to external packages\nand removing them from the kube-controller-manager, where they are today embedded. Once the extraction has been completed, existing users\nrunning older versions of Kubernetes need a process to migrate their existing clusters to use the new cloud-controller-manager component\nwith minimal risk.\n\nThis KEP proposes a mechanism in which HA clusters can safely migrate “cloud specific” controllers between the\nkube-controller-manager and the cloud-controller-manager via a shared resource lock between the two components. The pattern proposed\nin this KEP should be reusable by other components in the future if desired.\n\nThe migration mechanism outlined in this KEP should only be used for Kubernetes clusters that have _very_ strict requirements on control plane availability.\nIf a cluster can tolerate short intervals of downtime, it is recommended to update your cluster with in-tree cloud providers disabled, and then deploy\nthe respective out-of-tree cloud-controller-manager.\n\n### Goals\n\n* Define migration process for large scale, highly available clusters to migrate from the in-tree cloud provider mechnaism, to their out-of-tree equivalents.\n\n### Non-Goals\n\n* Removing cloud provider code from the core Kubernetes tree, this effort is separate and is covered in [KEP-removing-in-tree-providers](https://github.com/kubernetes/enhancements/blob/master/keps/sig-cloud-provider/20190125-removing-in-tree-providers.md)\n* Improving the scalability of controllers by running controllers across multiple components (with or without leader election).\n* Migrating cloud-based volume plugins to CSI. This is a separate effort led by SIG Storage. See [this proposal](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/csi-migration.md) for more details.\n\n## Proposal\n\nToday, the kube-controller-manager (KCM) and cloud-controller-manager (CCM) run independent of each other.\nThis means that both the KCM or the CCM can run the cloud specific control loops for a given cluster.\nFor a highly available control plane to migrate from running only the KCM to running both the KCM and\nthe CCM requires that only one process in the control plane is running the cloud specific controllers.\nThis becomes non-trivial when introducing the CCM that runs overlapping controllers with the KCM.\n\nFor environments that can tolerate downtime, the control plane can be taken down in order to\nreconfigure components to use the CCM, and then bring the control plane back up. This ensures that only\n1 component can be running the set of cloud controllers. For environments that have stricter requirements\nfor uptime, some level of coordination is required between the two components to ensure that upgrading\ncontrol planes does not result in running the same controller in more than one place while also accounting for version skew.\n\nIn order to coordinate the cloud-specific controllers across the KCM and the CCM, this KEP proposes a\n_primary_ and N configurable _secondary_ (a.k.a migration) leader election locks in the KCM and the CCM.\nThe primary lock represents the current leader election resource lock in the KCM and the CCM. The set of\nsecondary locks are defined by the cloud provider and run in parallel to the primary locks. For a migration\nlock defined by the cloud provider, the cloud provider also determines the set of controllers run within the\nmigration lock and the controller manager it should run in - either the CCM or the KCM.\n\nThe properties of the migration lock are:\n  * must have a unique name\n  * the set of controllers in the lock is immutable.\n  * no two migration locks should have overlapping controllers\n  * the controller manager where the lock runs can change across releases.\n  * for a minor release it should run exclusively in one type of controller manager - KCM or CCM.\n\nDuring migration, either the KCM or CCM may have multiple migration locks, though for performance reasons no more than 2 locks is recommended.\n\nLet's say we are migrating the service, route, and nodeipam controllers from the KCM to the CCM across Kubernetes versions, say v1.17 to v1.18.\nIn v1.17, the cloud provider would define a new migration lock called `cloud-network-controller-migration` which specifies those controllers to run\ninside the KCM (see Figure 1). As a result, in v1.17 those controllers would run in the KCM but under the `cloud-network-controller-migration` leader election.\nTo migrate to the CCM for v1.18, the cloud provider would update the `cloud-network-controller-migration` lock to now run in the CCM (see Figure 2).\nDuring a control plane upgrade, the cloud network controllers may still run in one of the KCMs that are still on v1.17. A 1.17 KCM holding the lock\nwill prevent any of the v1.18 CCMs from claiming the lock. When the current holder of the lock goes down, one of the controller managers eligible will acquire lock.\n\n\u003cbr/\u003e\n\u003cbr/\u003e\n\u003cbr/\u003e\n\n![example network controllers migration v1.17](/keps/sig-cloud-provider/images/migrating-cloud-controllers-v1-17.png)\n\u003cbr/\u003e\n**Figure 1**: Example of migrating cloud network controllers in v1.17\n\n\u003cbr/\u003e\n\u003cbr/\u003e\n\u003cbr/\u003e\n\n![example network controllers migration v1.18](/keps/sig-cloud-provider/images/migrating-cloud-controllers-v1-18.png)\n\u003cbr/\u003e\n**Figure 2**: Example of migrating cloud network controllers in v1.18\n\n\n\n### Implementation Details/Notes/Constraints [optional]\n\n#### Migration Configuration\n\nThe migration lock will be configured by defining new API types that will then be passed into the KCM and CCM.\n\n```go\n// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object\n// LeaderMigrationConfiguration provides versioned configuration for all migrating leader locks.\ntype LeaderMigrationConfiguration struct {\n\tmetav1.TypeMeta `json:\",inline\"`\n\n\t// LeaderName is the name of the resource under which the controllers should be run.\n\tLeaderName string `json:\"leaderName\"`\n\n\t// ControllerLeaders contains a list of migrating leader lock configurations\n\tControllerLeaders []ControllerLeaderConfiguration `json:\"controllerLeaders\"`\n}\n\n// ControllerLeaderConfiguration provides the configuration for a migrating leader lock.\ntype ControllerLeaderConfiguration struct {\n\t// Name is the name of the controller being migrated\n\t// E.g. service-controller, route-controller, cloud-node-controller, etc\n\tName string `json:\"name\"`\n\n\t// Component is the name of the component in which the controller should be running.\n\t// E.g. kube-controller-manager, cloud-controller-manager, etc\n\tComponent string `json:\"component\"`\n}\n```\n\n#### Component Flags\n\nThe LeaderMigrationConfiguration type will be read by the `kube-controller-manager` and the `cloud-controller-manager` via a new flag `--cloud-migration-config` which\naccepts a path to a file containing the LeaderMigrationConfiguration type in yaml.\n\n#### Example Walkthrough of Controller Migration\n\nThis is an example of how you would migrate all cloud controllers from the CCM to the KCM during a typical cluster version upgrade.\n\n##### Enable Leader Migration on Components\n\nFirst, define a LeaderMigrationConfiguration resource in a yaml file containing all known cloud controllers. The component name for each controller should be set to\nthe component where the controllers are currently running. Almost always this is the `kube-controller-manager`. The configuration file should look something like this:\n```yaml\nkind: LeaderMigrationConfiguration\napiVersion: v1alpha1\nleaderName: cloud-controllers-migration\ncontrollerLeaders:\n  - name: route-controller\n    component: kube-controller-manager\n  - name: service-controller\n    component: kube-controller-manager\n  - name: cloud-node-controller\n    component: kube-controller-manager\n  - name: cloud-nodelifecycle-controller\n    component: kube-controller-manager\n```\n\nSave the leader migration configuration file somewhere, for this example we'll use `/etc/kubernetes/cloud-controller-migration.yaml`.\nNow update the kube-controller-manager to set `--cloud-migration-config /etc/kubernetes/cloud-controller-migration.yaml`.\n\n##### Deploy the CCM\n\nNow deploy the CCM on your cluster but ensure it also has the `--cloud-migration-config` flag set, using the same config file you used for the KCM above.\n\nHow the CCM is deployed is out of scope for this KEP, refer to the cloud provider's documentation on how to do this.\n\n#####  Update Leader Migration Config on Upgrade\n\nTo migrate controllers from the KCM to the CCM, update the component field from `kube-controller-manager` to `cloud-controller-manager` on every control plane node prior to\nupgrading the node. If you are replacing nodes on upgrade, ensure new nodes set the `component` field to `cloud-controller-manager`. The new config file should look like this:\n```yaml\nkind: LeaderMigrationConfiguration\napiVersion: v1alpha1\nleaderName: cloud-controllers-migration\ncontrollerLeaders:\n  - name: route-controller\n    component: cloud-controller-manager\n  - name: service-controller\n    component: cloud-controller-manager\n  - name: cloud-node-controller\n    component: cloud-controller-manager\n  - name: cloud-nodelifecycle-controller\n    component: cloud-controller-manager\n```\n\nNOTE: During upgrade, it is acceptable for control plane nodes to specify different component names for each controller as long as the `leaderName` field is the same across nodes.\n\n##### Disable Leader Migration\n\nOnce all controllers are migrated to the desired component:\n* disable the cloud provider in the `kube-controller-manager` (set `--cloud-provider=external`)\n* disable leader migration on the `kube-controller-manager` and `cloud-controller-manager` by unsetting the `--cloud-migration-config` field.\n\n### Risks and Mitigations\n\n* Increased apiserver load due to new leader election resource per migration configuration.\n* User error could result in cloud controllers not running in any component at all.\n\n### Graduation Criteria\n\n##### Alpha -\u003e Beta Graduation\n\nLeader migration configuration is tested end-to-end on at least 2 cloud providers.\n\n##### Beta -\u003e GA Graduation\n\nLeader migration configuration works on all in-tree cloud providers.\n\n### Upgrade / Downgrade Strategy\n\nSee [Example Walkthrough of Controller Migration](#example-walkthrough-of-controller-migratoin) for upgrade strategy.\nClusters can be downgraded and migration can be disabled by reversing the steps in the upgrade strategy assuming the behavior of each controller\ndoes not change incompatibly across those versions.\n\n### Version Skew Strategy\n\nVersion skew is handled as long as the leader name is consistent across all control plane nodes during upgrade.\n\n## Implementation History\n\n- 07-25-2019 `Summary` and `Motivation` sections were merged signaling SIG acceptance\n- 01-21-2019  Implementation details are proposed to move KEP to `implementable` state.\n"
+  },
+  {
+    "id": "5eda7a210717d8706f02c0ab27834f6a",
+    "title": "Building Kubernetes Without In-Tree Cloud Providers",
+    "authors": ["@BenTheElder"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": ["sig-release", "sig-testing"],
+    "reviewers": ["@spiffxp", "@cheftako", "@andrewsykim", "@stephenaugustus"],
+    "approvers": ["@cheftako", "@andrewsykim", "@spiffxp", "@stephenaugustus"],
+    "editor": "TBD",
+    "creationDate": "2019-07-29",
+    "lastUpdated": "2019-07-29",
+    "status": "implementable",
+    "seeAlso": [
+      "/keps/sig-cloud-provider/20190125-removing-in-tree-providers.md",
+      "/keps/sig-cloud-provider/20180530-cloud-controller-manager.md"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Building Kubernetes Without In-Tree Cloud Providers\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories [optional]](#user-stories-optional)\n    - [Story 1](#story-1)\n    - [Story 2](#story-2)\n    - [Story 3](#story-3)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Drawbacks [optional]](#drawbacks-optional)\n- [Alternatives [optional]](#alternatives-optional)\n- [Infrastructure Needed [optional]](#infrastructure-needed-optional)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [x] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [x] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [x] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [x] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [x] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nThis proposal outlines a plan to enable building Kubernetes without the in-tree\ncloud providers in preparation for [removing them entirely](keps/sig-cloud-provider/20190125-removing-in-tree-providers.md).\n\n## Motivation\n\nThe in tree cloud-provider implementations are being [removed](keps/sig-cloud-provider/20190125-removing-in-tree-providers.md) in the future, this involves a large amount\nof code that is used in many places in tree. In order to prepare for this eventuality\nit would be helpful to see what that removal entails exactly and verify that Kubernetes\nwill continue to function correctly. Doing so is a bit tricky without ensuring\nthat the in-tree provider code is not being used via some unexpected side-channel\n(such as `init()` methods). Building binaries without the in-tree cloud provider\npackages would allow us to verify this and additionally provide experimentally\nsmaller / cheaper binaries for parties interested in out of tree provider or \nno provider based clusters.\n\n### Goals\n\n- Enable building Kubernetes without in-tree cloud providers and without forking\n  - Enable testing out of tree providers with a simulation of the future removal of the in-tree code.\n  - Enable experimentation with cloud-provider-less clusters\n\n### Non-Goals\n\n- Building the out of tree providers\n- Changing the official Kubernetes release builds\n- Building the e2e tests \n  - Decoupling cloud providers is a larger problem there and not necessary to test out-of-tree providers or build smaller binaries\n- Mechanisms for migrating to out of tree providers\n  - CSI Migration for in-tree Volumes is already underway in SIG Storage\n  - External Credential Providers is being written / solved in another KEP ([#541](https://github.com/kubernetes/enhancements/issues/541))\n  - CCMs and overall scope for moving out of tree is in [removing-in-tree-providers](keps/sig-cloud-provider/20190125-removing-in-tree-providers.md)\n\n## Proposal\n\nWe will add a [build constraints](https://golang.org/pkg/go/build/#hdr-Build_Constraints)\nto the cloud provider code for a pseudo \"build tag\" specifying not to include\nany in-tree provider code. This will allow compiling the binaries as normal today\nand simulating the removal of this code by specifying the tag at build time and\ntriggering the build constraints on the files in these packages.\n\nSome small adjustments may be necessary to the code base to ensure that the\nother packages can build without depending on these packages.\n\nA prototype is available in [kubernetes/kubernetes#80353](https://github.com/kubernetes/kubernetes/pull/80353).\n\nTo ensure that this continues to function we will add CI building in this mode,\nand CI running end to end tests against it (see the test plan).\n\n### User Stories [optional]\n\n#### Story 1\n\nAs an out of tree cloud provider implementer, I want to develop and test against\nKubernetes without the in tree providers.\n\nKubernetes out of tree cloud provider developers will be able to build Kubernetes\nin this mode and build \u0026 test their cloud-controller-manager implementations and\nassociated tooling against this build in preparation for the actual hard removal\nof the in-tree providers.\n\n#### Story 2\n\nAs a developer working to replace an in-tree provider with an out-of-tree provider,\nI am attempting to validate that I work with KAS/KCM/Kubelet which do not have \n(my) in-tree cloud-provider compiled in and have successfully migrated all the \nfunctionality I need to CCM/CSI/... Using this build ensures the relevant \nfunctionality is not in KAS/KCM/Kubelet. It also allows me to work with \nsmaller binaries.\n\n#### Story 3\n\nAs a [kind](https://github.com/kubernetes-sigs/kind) developer / user I want to \nuse Kubernetes binaries without cloud providers for local clusters.\n\nDevelopers and users will be able to build local clusters leveraging this mode\nto not pay for cloud providers they are unable to use.\n\n### Implementation Details/Notes/Constraints [optional]\n\nThis is implemented using a synthetic `nolegacyproviders` tag in go build\nconstraints on the relevant sources. If `GOFLAGS=-tags=nolegacyproviders` is set\nthen the legacy cloud provider pacakges will be excluded from the build.\n\nIn order to make this work the following additional changes are made:\n\n- Packages that we fully exclude (the legacy provider packages) _must_ contain\na `doc.go` file or any other file that does NOT contain any code or build \nconstraints. Go will not allow \"building\" a package without any files passing\nthe constraints, however it will happily build a package with no actual methods\n/ variables / ...\n\n- A few locations in the code do not properly use the cloud provider interface\n(instead, importing the cloud provider packages directly),\nsome of these must be updated with both a \"with provider\" version and a\n\"without provider\" version broken out of the existing code. In particular this \nincludes the in-tree volumes until CSI migration is standard, and the GCE IPAM\nlogic in kube-controller-manager.\n  - Note that the nodeIpamController GCE IPAM logic is slated for removal (see [the cloud controller manager KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-cloud-provider/20180530-cloud-controller-manager.md))\n\nIn particular this adds tags / constraints to:\n- `staging/src/k8s.io/legacy-cloud-providers/*` (constraints on all the providers)\n- `pkg/cloudprovider` (constraints on importing the providers)\n- `pkg/volumes/*`, `cmd/kubelet` (versions with and without the imported providers for in-tree volumes)\n- `pkg/controller/nodeipam`, `cmd/kube-apiserver`, `cmd/kube-controller-manager` (with and without GCE IPAM)\n\n`test/*` is punted to a future follow up, and credential providers are punted\nto [the external credential provider KEP](https://github.com/kubernetes/enhancements/pull/1137).\n\n### Risks and Mitigations\n\nThis is only developer facing, however we will need to ensure that these tags\nstay up to date if we want this build mode to continue to work (the normal\nbuild mode should work by default without any additional maintenance).\n\nTo ensure this continues to work we can mitigate by:\n\n- verify in CI that the cloud provider packages have boilerplate including the\nbuild constraints\n- build in this mode in CI to ensure that the build succeeds\n\n## Design Details\n\n### Test Plan\n\nWe will add CI to ensure that we can build with this mode enabled.\n\nAdditionally, we can add CI to ensure that clusters can actually be started in\nthis mode.\n\nInitially, [kind](https://github.com/kubernetes-sigs/kind) can be used to ensure\nthat Kubernetes works without the providers, in the future we can extend this\nCI to out-of-tree providers combined with this build mode as their CI is spun up.\n\n### Graduation Criteria\n\n##### Alpha -\u003e Beta Graduation\n\nLikely unnecessary, as we will eventually remove the in-tree provider code entirely for [removing-in-tree-providers](keps/sig-cloud-provider/20190125-removing-in-tree-providers.md).\nThis is also not a user facing change.\n\n##### Beta -\u003e GA Graduation\n\nLikely unnecessary, as we will eventually remove the in-tree provider code entirely for  [removing-in-tree-providers](keps/sig-cloud-provider/20190125-removing-in-tree-providers.md).\nThis is also not a user facing change.\n\nFinal graduation can be considered to be when the cloud provider code is actually\nremoved from the Kubernetes source tree, at which point this work will be complete.\n\n### Upgrade / Downgrade Strategy\n\nN/A ?\n\n### Version Skew Strategy\n\nN/A ?\n\n## Implementation History\n\n- original prototype [kubernetes/kubernetes#80353](https://github.com/kubernetes/kubernetes/pull/80353)\n- original KEP PR [kubernetes/enhancements#1180](https://github.com/kubernetes/enhancements/pull/1180)\n\n## Drawbacks [optional]\n\nThis does require maintaining these tags / constraints for the providerless build,\nhowever in the default mode without our pseudo-tag the code will build as today\nand require zero additional maintenance to function. As in-tree providers are\nrelatively stable and expected not to gain new features, this should require\nminimal effort and can be automated to a limited extent.\n\n## Alternatives [optional]\n\nWe could simply wait for the in-tree providers to be removed entirely, however\nthis may not provide sufficient tools to adequately prepare.\n\nThere is also a risk that cloud providers would each need to duplicate this\nwork to test cloud-provider free Kubernetes for their out of tree provider.\n\nWe could attempt to create a branch/PR with those changes in them. \nHowever the in-tree providers are not guaranteed to exit at the same time. \nSo the branch/PR might have to be kept for a long period of time. \nIn addition to being expensive the maintain such a PR/branch, it would obfuscate\nthe effort. So developers would end up changing CP related code and have little\n/ no visibility that their changes were CP related.\n\n## Infrastructure Needed [optional]\n\nNone?\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n"
+  },
+  {
+    "id": "ecfd0a5d7708a87c1953f2695b13d05f",
+    "title": "Azure Availability Zones",
+    "authors": ["@feiskyer"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": ["sig-storage"],
+    "reviewers": ["@khenidak", "@colemickens"],
+    "approvers": ["@brendanburns"],
+    "editor": "@feiskyer",
+    "creationDate": "2018-07-11",
+    "lastUpdated": "2018-09-29",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Azure Availability Zones\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Scopes and Non-scopes](#scopes-and-non-scopes)\n  - [Scopes](#scopes)\n  - [Non-scopes](#non-scopes)\n- [AZ label format](#az-label-format)\n- [Cloud provider options](#cloud-provider-options)\n- [Node registration](#node-registration)\n  - [Get by instance metadata](#get-by-instance-metadata)\n  - [Get by Go SDK](#get-by-go-sdk)\n- [LoadBalancer and PublicIP](#loadbalancer-and-publicip)\n- [AzureDisk](#azuredisk)\n  - [PVLabeler interface](#pvlabeler-interface)\n  - [PersistentVolumeLabel admission controller](#persistentvolumelabel-admission-controller)\n  - [StorageClass](#storageclass)\n- [Implementation History](#implementation-history)\n- [Appendix](#appendix)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis proposal aims to add [Azure Availability Zones (AZ)](https://azure.microsoft.com/en-us/global-infrastructure/availability-zones/) support to Kubernetes.\n\n## Scopes and Non-scopes\n\n### Scopes\n\nThe proposal includes required changes to support availability zones for various functions in Azure cloud provider and AzureDisk volumes:\n\n- Detect availability zones automatically when registering new nodes (by kubelet or node controller) and node's label `failure-domain.beta.kubernetes.io/zone` will be replaced with AZ instead of fault domain\n- LoadBalancer and PublicIP will be provisioned with zone redundant\n- `GetLabelsForVolume` interface will be implemented for Azure managed disks so that PV label controller in cloud-controller-manager can appropriately add `Labels` and `NodeAffinity` to the Azure managed disk PVs. Additionally, `PersistentVolumeLabel` admission controller will be enhanced to achieve the same for Azure managed disks.\n- Azure Disk's `Provision()` function will be enhanced to take into account the zone of the node as well as `allowedTopologies` when determining the zone to create a disk in.\n\n\u003e Note that unlike most cases, fault domain and availability zones mean different on Azure:\n\u003e\n\u003e - A Fault Domain (FD) is essentially a rack of servers. It consumes subsystems like network, power, cooling etc.\n\u003e - Availability Zones are unique physical locations within an Azure region. Each zone is made up of one or more data centers equipped with independent power, cooling, and networking.\n\u003e\n\u003e An Availability Zone in an Azure region is a combination of a fault domain and an update domain (Same like FD, but for updates. When upgrading a deployment, it is carried out one update domain at a time). For example, if you create three or more VMs across three zones in an Azure region, your VMs are effectively distributed across three fault domains and three update domains.\n\n### Non-scopes\n\nProvisioning Kubernetes masters and nodes with availability zone support is not included in this proposal. It should be done in the provisioning tools (e.g. acs-engine). Azure cloud provider will auto-detect the node's availability zone if `availabilityZones` option is configured for the Azure cloud provider.\n\n## AZ label format\n\nCurrently, Azure nodes are registered with label `failure-domain.beta.kubernetes.io/zone=faultDomain`.\n\nThe format of fault domain is numbers (e.g. `1` or `2`), which is in same format with AZ (e.g. `1` or `3`). If AZ is using same format with faultDomain, then there'll be scheduler issues for clusters with both AZ and non-AZ nodes. So AZ will use a different format in kubernetes: `\u003cregion\u003e-\u003cAZ\u003e`, e.g. `centralus-1`.\n\nThe AZ label will be applied in multiple Kubernetes resources, e.g.\n\n- Nodes\n- AzureDisk PersistentVolumes\n- AzureDisk StorageClass\n\n## Cloud provider options\n\nBecause only standard load balancer is supported with AZ, it is a prerequisite to enable AZ for the cluster.\n\nStandard load balancer has been added in Kubernetes v1.11, related options include:\n\n| Option                      | Default | **AZ Value**  | Releases | Notes                                 |\n| loadBalancerSku             | basic   | **standard**  | v1.11    | Enable standard LB                    |\n| excludeMasterFromStandardLB | true    | true or false | v1.11    | Exclude master nodes from LB backends |\n\nThese options should be configured in Azure cloud provider configure file (e.g. `/etc/kubernetes/azure.json`):\n\n```json\n{\n    ...,\n    \"loadBalancerSku\": \"standard\",\n    \"excludeMasterFromStandardLB\": true\n}\n```\n\nNote that with standard SKU LoadBalancer, `primaryAvailabitySetName` and `primaryScaleSetName` is not required because all available nodes (with configurable masters via `excludeMasterFromStandardLB`) are added to LoadBalancer backend pools.\n\n## Node registration\n\nWhen registering new nodes, kubelet (with build in cloud provider) or node controller (with external cloud provider) automatically adds labels to them with region and zone information:\n\n- Region: `failure-domain.beta.kubernetes.io/region=centralus`\n- Zone: `failure-domain.beta.kubernetes.io/zone=centralus-1`\n\n```sh\n$ kubectl get nodes --show-labels\nNAME                STATUS    AGE   VERSION    LABELS\nkubernetes-node12   Ready     6m    v1.11      failure-domain.beta.kubernetes.io/region=centralus,failure-domain.beta.kubernetes.io/zone=centralus-1,...\n```\n\nAzure cloud providers sets fault domain for label `failure-domain.beta.kubernetes.io/zone` today. With AZ enabled, we should set the node's availability zone instead. To keep backward compatibility and distinguishing from fault domain, `\u003cregion\u003e-\u003cAZ\u003e` is used here.\n\nThe node's zone could get by ARM API or instance metadata. This will be added in  `GetZoneByProviderID()` and `GetZoneByNodeName()`.\n\n### Get by instance metadata\n\nThis method is used in kube-controller-manager.\n\n```sh\n# Instance metadata API should be upgraded to 2017-12-01.\n$ curl -H Metadata:true \"http://169.254.169.254/metadata/instance/compute/zone?api-version=2017-12-01\u0026format=text\"\n2\n```\n\n### Get by Go SDK\n\nThis method is used in cloud-controller-manager.\n\nNo `zones` property is included in `VirtualMachineScaleSetVM` yet in Azure Go SDK (including latest 2018-04-01 compute API).\n\nWe need to ask Azure Go SDK to add `zones` for `VirtualMachineScaleSetVM`. Opened the issue https://github.com/Azure/azure-sdk-for-go/issues/2183 for tracking it.\n\n\u003e Note: there's already `zones` property in `VirtualMachineScaleSet`, `VirtualMachine` and `Disk`.\n\n## LoadBalancer and PublicIP\n\nLoadBalancer with standard SKU will be created and all available nodes (including VirtualMachines and VirtualMachineScaleSetVms, together with optional masters configured via excludeMasterFromStandardLB) are added to LoadBalancer backend pools.\n\nPublicIPs will also be created with standard SKU, and they are zone redundant by default.\n\nNote that zonal PublicIPs are not supported. We may add this easily if there’re clear use-cases in the future.\n\n## AzureDisk\n\nWhen Azure managed disks are created, the `PersistentVolumeLabel` admission controller or PV label controller automatically adds zone labels and node affinity to them. The scheduler (via `VolumeZonePredicate` or `PV.NodeAffinity`) will then ensure that pods that claim a given volume are only placed into the same zone as that volume, as volumes cannot be attached across zones. In addition, admission controller\n\nNote that\n\n- Only managed disks are supported. Blob disks don't support availability zones on Azure.\n- Node affinity is enabled by feature gate `VolumeScheduling`.\n\n### PVLabeler interface\n\nTo setup AzureDisk's zone label correctly (required by cloud-controller-manager's PersistentVolumeLabelController), Azure cloud provider's [PVLabeler](https://github.com/kubernetes/kubernetes/blob/master/pkg/cloudprovider/cloud.go#L212) interface should be implemented:\n\n```go\n// PVLabeler is an abstract, pluggable interface for fetching labels for volumes\ntype PVLabeler interface {\n    GetLabelsForVolume(ctx context.Context, pv *v1.PersistentVolume) (map[string]string, error)\n}\n```\n\nIt should return the region and zone for the AzureDisk, e.g.\n\n- `failure-domain.beta.kubernetes.io/region=centralus`\n- `failure-domain.beta.kubernetes.io/zone=centralus-1`\n\nso that the PV will be created with labels:\n\n```sh\n$ kubectl get pv --show-labels\nNAME           CAPACITY   ACCESSMODES   STATUS    CLAIM            REASON    AGE       LABELS\npv-managed-abc 5Gi        RWO           Bound     default/claim1             46s       failure-domain.beta.kubernetes.io/region=centralus,failure-domain.beta.kubernetes.io/zone=centralus-1\n```\n\n### PersistentVolumeLabel admission controller\n\nCloud provider's `PVLabeler` interface is only applied when cloud-controller-manager is used. For build in Azure cloud provider, [PersistentVolumeLabel](https://github.com/kubernetes/kubernetes/blob/master/plugin/pkg/admission/storage/persistentvolume/label/admission.go) admission controller should also updated with AzureDisk support, so that new PVs could also be applied with above labels.\n\n```go\nfunc (l *persistentVolumeLabel) Admit(a admission.Attributes) (err error) {\n    ...\n    if volume.Spec.AzureDisk != nil {\n        labels, err := l.findAzureDiskLabels(volume)\n        if err != nil {\n            return admission.NewForbidden(a, fmt.Errorf(\"error querying AzureDisk volume %s: %v\", volume.Spec.AzureDisk.DiskName, err))\n        }\n        volumeLabels = labels\n    }\n    ...\n}\n```\n\n\u003e Note: the PersistentVolumeLabel admission controller will be deprecated, and cloud-controller-manager is preferred after its GA (probably v1.13 or v1.14).\n\n### StorageClass\n\nNote that the above interfaces are only applied to AzureDisk persistent volumes, not StorageClass. For AzureDisk StorageClass, we should add a few new options for zone-aware and [topology-aware](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/volume-topology-scheduling.md) provisioning. The following three new options will be added in AzureDisk StorageClass:\n\n- `zoned`: indicates whether new disks are provisioned with AZ. Default is `true`.\n- `zone` and `zones`: indicates which zones should be used to provision new disks (zone-aware provisioning). Only can be set if `zoned` is not false and `allowedTopologies` is not set.\n- `allowedTopologies`: indicates which topologies are allowed for [topology-aware](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/volume-topology-scheduling.md) provisioning. Only can be set if `zoned` is not false and `zone`/`zones` are not set.\n\nAn example of zone-aware provisioning storage class is:\n\n```yaml\napiVersion: storage.k8s.io/v1\nkind: StorageClass\nmetadata:\n  annotations:\n  labels:\n    kubernetes.io/cluster-service: \"true\"\n  name: managed-premium\nparameters:\n  kind: Managed\n  storageaccounttype: Premium_LRS\n  # only one of zone and zones are allowed\n  zone: \"centralus-1\"\n  # zones: \"centralus-1,centralus-2,centralus-3\"\nprovisioner: kubernetes.io/azure-disk\n```\n\nAnother example of topology-aware provisioning storage class is:\n\n```yaml\napiVersion: storage.k8s.io/v1\nkind: StorageClass\nmetadata:\n  annotations:\n  labels:\n    kubernetes.io/cluster-service: \"true\"\n  name: managed-premium\nparameters:\n  kind: Managed\n  storageaccounttype: Premium_LRS\nprovisioner: kubernetes.io/azure-disk\nallowedTopologies:\n- matchLabelExpressions:\n  - key: failure-domain.beta.kubernetes.io/zone\n    values:\n    - centralus-1\n    - centralus-2\n```\n\nAzureDisk can only be created with one specific zone, so if multiple zones are specified in the storage class, then new disks will be provisioned with zone chosen by following rules:\n\n- If `DynamicProvisioningScheduling` is enabled and `VolumeBindingMode: WaitForFirstConsumer` is specified in the storage class, zone of the disk should be set to the zone of the node passed to `Provision()`. Specifying zone/zones in storage class should be considered an error in this scenario.\n- If `DynamicProvisioningScheduling` is enabled and `VolumeBindingMode: WaitForFirstConsumer` is not specified in StorageClass, zone of disk should be chosen from `allowedTopologies` or zones depending on which is specified. Specifying both `allowedTopologies` and `zones` should lead to error.\n- If `DynamicProvisioningScheduling` is disabled and `zones` are specified, then the zone maybe arbitrarily chosen as specified by arbitrarily choosing from the zones specified in the storage class.\n- If `DynamicProvisioningScheduling` is disabled and no zones are specified and `zoned` is `true`, then new disks will be provisioned with zone chosen by round-robin across all active zones, which means\n  - If there are no zoned nodes, then an `no zoned nodes` error will be reported\n  - Zoned AzureDisk will only be provisioned when there are zoned nodes\n  - If there are multiple zones, then those zones are chosen by round-robin\n\nNote that\n\n- active zones means there're nodes in that zone.\n- there are risks if the cluster is running with both zoned and non-zoned nodes. In such case, zoned AzureDisk can't be attached to non-zoned nodes. So\n  - new pods with zoned AzureDisks are always scheduled to zoned nodes\n  - old pods using non-zoned AzureDisks can't be scheduled to zoned nodes\n\nSo if users are planning to migrate workloads to zoned nodes, old AzureDisks should be recreated (probably backup first and restore to the new one).\n\n## Implementation History\n\n- [kubernetes#66242](https://github.com/kubernetes/kubernetes/pull/66242): Adds initial availability zones support for Azure nodes.\n- [kubernetes#66553](https://github.com/kubernetes/kubernetes/pull/66553): Adds avaialability zones support for Azure managed disks.\n- [kubernetes#67121](https://github.com/kubernetes/kubernetes/pull/67121): Adds DynamicProvisioningScheduling and VolumeScheduling support for Azure managed disks.\n- [cloud-provider-azure#57](https://github.com/kubernetes/cloud-provider-azure/pull/57): Adds documentation for Azure availability zones.\n\n## Appendix\n\nKubernetes will automatically spread the pods in a replication controller or service across nodes in a single-zone cluster (to reduce the impact of failures).\n\nWith multiple-zone clusters, this spreading behavior is extended across zones (to reduce the impact of zone failures.) (This is achieved via `SelectorSpreadPriority`). This is a best-effort placement, and so if the zones in your cluster are heterogeneous (e.g. different numbers of nodes, different types of nodes, or different pod resource requirements), this might prevent perfectly even spreading of your pods across zones. If desired, you can use homogeneous zones (same number and types of nodes) to reduce the probability of unequal spreading.\n\nThere's also some [limitations of availability zones of various Kubernetes functions](https://kubernetes.io/docs/setup/multiple-zones/#limitations), e.g.\n\n- No zone-aware network routing\n- Volume zone-affinity will only work with a `PersistentVolume`, and will not work if you directly specify an AzureDisk volume in the pod spec.\n- Clusters cannot span clouds or regions (this functionality will require full federation support).\n- StatefulSet volume zone spreading when using dynamic provisioning is currently not compatible with pod affinity or anti-affinity policies.\n- If the name of the StatefulSet contains dashes (“-”), volume zone spreading may not provide a uniform distribution of storage across zones.\n- When specifying multiple PVCs in a Deployment or Pod spec, the StorageClass needs to be configured for a specific, single zone, or the PVs need to be statically provisioned in a specific zone. Another workaround is to use a StatefulSet, which will ensure that all the volumes for a replica are provisioned in the same zone.\n\nSee more at [running Kubernetes in multiple zones](https://kubernetes.io/docs/setup/multiple-zones/).\n"
+  },
+  {
+    "id": "685a5066ab87a572f7221be7e07f61ee",
+    "title": "Cross resource group nodes",
+    "authors": ["@feiskyer"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": null,
+    "reviewers": ["@khenidak", "@justaugustus"],
+    "approvers": ["@brendanburns"],
+    "editor": "@feiskyer",
+    "creationDate": "2018-08-09",
+    "lastUpdated": "2018-09-29",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Cross resource group nodes\n\n## Table of Contents\n\n\u003c!-- TOC --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Assumptions](#assumptions)\n  - [Non-Goals](#non-goals)\n- [Design](#design)\n- [Implementation](#implementation)\n  - [Cross-RG nodes](#cross-rg-nodes)\n  - [On-prem nodes](#on-prem-nodes)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n\u003c!-- /TOC --\u003e\n\n## Summary\n\nThis KEP aims to add support for cross resource group (RG) and on-prem nodes to the Azure cloud provider.\n\n## Motivation\n\nToday, the Azure cloud provider only supports nodes from a specified RG (which is set in the cloud provider configuration file). For nodes in a different RG, Azure cloud provider reports `InstanceNotFound` error and thus they would be removed by controller manager. The same holds true for on-prem nodes.\n\nWith managed clusters, like [AKS](https://docs.microsoft.com/en-us/azure/aks/), there is limited access to configure nodes. There are instances where users may need to customize nodes in ways that are not possible in a managed service. This document proposes support for joining arbitrary nodes to a cluster and the required changes to make in both the Azure cloud provider and provisioned setups, which include:\n\n- Provisioning tools should setup kubelet with required labels (e.g. via `--node-labels`)\n- Azure cloud provider would fetch RG from those labels and then get node information based on that\n\n### Assumptions\n\nWhile new nodes (either from different RGs or on-prem) would be supported in this proposal, not all features would be supported for them. For example, AzureDisk will not work for on-prem nodes.\n\nThis proposal makes following assumptions for those new nodes:\n\n- Nodes are in same region and set with required labels (as clarified in the following design part)\n- Nodes will not be part of the load balancer managed by cloud provider\n- Both node and container networking are properly configured\n- AzureDisk is supported for Azure cross-RG nodes, but not for on-prem nodes\n\nIn addition, feature gate [ServiceNodeExclusion](https://github.com/kubernetes/kubernetes/blob/master/pkg/features/kube_features.go#L174) must also be enabled for Kubernetes cluster.\n\n### Non-Goals\n\nNote that provisioning the Kubernetes cluster, setting up networking and provisioning new nodes are out of this proposal scope. Those could be done by external provisioning tools (e.g. acs-engine).\n\n## Design\n\nInstance metadata is a general way to fetch node information for Azure, but it doesn't work if cloud-controller-manager is used (`kubelet --cloud-provider=external`). So it won't be used in this proposal. Instead, the following labels are proposed for providing required information:\n\n- `alpha.service-controller.kubernetes.io/exclude-balancer=true`, which is used to exclude the node from load balancer. Required.\n- `kubernetes.azure.com/resource-group=\u003crg-name\u003e`, which provides external RG and is used to get node information. Required for cross-RG nodes.\n- `kubernetes.azure.com/managed=true|false`, which indicates whether a node is on-prem or not. Required for on-prem nodes with `false` value.\n\nWhen initializing nodes, these two labels should be set for kubelet by provisioning tools, e.g.\n\n```sh\n# For cross-RG nodes\nkubelet --node-labels=alpha.service-controller.kubernetes.io/exclude-balancer=true,kubernetes.azure.com/resource-group=\u003crg-name\u003e ...\n\n# For on-prem nodes\nkubelet --node-labels=alpha.service-controller.kubernetes.io/exclude-balancer=true,kubernetes.azure.com/managed=false ...\n```\n\nNode label `alpha.service-controller.kubernetes.io/exclude-balancer=true` has already been supported in Kubernetes, and it is controlled by feature gate `ServiceNodeExclusion`. Cluster admins should ensure the feature gate `ServiceNodeExclusion` opened when provisioning the cluster.\n\nNote that\n\n- Azure resource group name supports a [wider range of valid characters](https://docs.microsoft.com/en-us/azure/architecture/best-practices/naming-conventions#naming-rules-and-restrictions) than [Kubernetes labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set). **Only kubernetes labels compatible resource group names** are supported, which must be 63 characters or less and must be empty or begin and end with an alphanumeric character (`[a-z0-9A-Z]`) with dashes (`-`), underscores (`_`), dots (`.`), and alphanumerics between.\n- If the label `kubernetes.azure.com/managed` is not provided, then Azure cloud provider will assume the node to be managed.\n\n## Implementation\n\n### Cross-RG nodes\n\nCross-RG nodes should register themselves with required labels together with cloud provider:\n\n- `--cloud-provider=azure` when using kube-controller-manager\n- `--cloud-provider=external` when using cloud-controller-manager\n\nFor example,\n\n```sh\nkubelet ... \\\n  --cloud-provider=azure \\\n  --cloud-config=/etc/kubernetes/azure.json \\\n  --node-labels=alpha.service-controller.kubernetes.io/exclude-balancer=true,kubernetes.azure.com/resource-group=\u003crg-name\u003e\n```\n\n[LoadBalancer](https://github.com/kubernetes/kubernetes/blob/master/pkg/cloudprovider/cloud.go#L92) is not required for cross-RG nodes, hence only following features will be implemented for them:\n\n- [Instances](https://github.com/kubernetes/kubernetes/blob/master/pkg/cloudprovider/cloud.go#L121)\n- [Zones](https://github.com/kubernetes/kubernetes/blob/master/pkg/cloudprovider/cloud.go#L194)\n- [Routes](https://github.com/kubernetes/kubernetes/blob/master/pkg/cloudprovider/cloud.go#L169)\n- [Azure managed disks](https://github.com/kubernetes/kubernetes/tree/master/pkg/volume/azure_dd)\n\nMost operations of those features are similar with existing nodes, except the RG name. The existing nodes are using RG from cloud provider configure, while cross-RG nodes will get RG from node label `kubernetes.azure.com/resource-group=\u003crg-name\u003e`.\n\nTo achieve this， [Informers](https://github.com/kubernetes/kubernetes/blob/master/pkg/cloudprovider/cloud.go#L52-L55) will be used to get node labels and then their RGs will be cached in `nodeResourceGroups map[string]string`.\n\n```go\ntype Cloud struct {\n   ...\n   // nodeResourceGroups is a mapping from Node's name to resource group name.\n   // It will be updated by the nodeInformer.\n   nodeResourceGroups map[string]string\n}\n```\n\n### On-prem nodes\n\nOn-prem nodes are different from Azure nodes, all Azure coupled features (including Instances, LoadBalancer, Zones, Routes and Azure managed disks) are not supported for them. To prevent the node being deleted, Azure cloud provider will always assumes the node existing and use providerID in format `azure://\u003cnode-name\u003e`.\n\nOn-prem nodes should register themselves with labels `alpha.service-controller.kubernetes.io/exclude-balancer=true` and `kubernetes.azure.com/managed=false`, e.g.\n\n```sh\nkubelet --node-labels=alpha.service-controller.kubernetes.io/exclude-balancer=true,kubernetes.azure.com/managed=false ...\n```\n\nBecause AzureDisk is also not supported, and we don't expect Pods using AzureDisk being scheduled to on-prem nodes, a new taint `kubernetes.azure.com/managed:NoSchedule` will be added for those nodes.\n\nTo run workloads on them, nodeSelector and tolerations should be provided. For example,\n\n```yaml\napiVersion: v1\nkind: Pod\nmetadata:\n name: nginx\nspec:\n containers:\n - image: nginx\n   name: nginx\n   ports:\n   - containerPort: 80\n     name: http\n     protocol: TCP\n dnsPolicy: ClusterFirst\n nodeSelector:\n   kubernetes.azure.com/resource-group: on-prem\n tolerations:\n - key: kubernetes.azure.com/managed\n   effect: NoSchedule\n```\n\n## Graduation Criteria\n\n- [ ] Feature complete\n- [ ] Documentation should be added\n- [ ] E2e tests should cover the feature\n\n## Implementation History\n\n- [kubernetes#67604](https://github.com/kubernetes/kubernetes/pull/67604): Adds initial support for Azure cross resource group nodes.\n- [kubernetes#67984](https://github.com/kubernetes/kubernetes/pull/67984): Adds unmanaged nodes support for Azure cloud provider.\n- [cloud-provider-azure#58](https://github.com/kubernetes/cloud-provider-azure/pull/58): Adds documentation for Azure cross resource group nodes.\n\n## Alternatives\n\nAnnotations, additional cloud provider options and querying directly from Azure API are three alternatives ways to provide resource group information. They are not preferred because\n\n- Kubelet doesn't support registering itself with annotations, so it requires admin to annotate the node afterward. The extra steps add complexity for cluster operations.\n- Cloud provider options are not flexible compared to labels and annotations. It needs configure file updates and controller manager restarts if unknown resource groups are used for new nodes.\n- Querying node information directly from Azure API is also not feasible because that would need list all resource groups, all virtual machine scale sets and all virtual machines. The operation is time consuming and easy to hit rate limits.\n"
+  },
+  {
+    "id": "70d1b3b6b54f68bbc9183415759315dd",
+    "title": "Support Out-of-Tree Azure Cloud Provider",
+    "authors": ["@andrewsykim", "@dstrebel", "@feiskyer"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": null,
+    "reviewers": ["@dstrebel", "@justaugustus", "@khenidak", "@feiskyer"],
+    "approvers": ["@feiskyer", "@khenidak", "@hogepodge", "@jagosan"],
+    "editor": "@feiskyer",
+    "creationDate": "2019-01-29",
+    "lastUpdated": "2020-01-18",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Supporting Out-of-Tree Azure Cloud Provider\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Documentation](#documentation)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n  - [Risks and Mitigation](#risks-and-mitigation)\n    - [API throttling](#api-throttling)\n    - [Azure credential provider](#azure-credential-provider)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Technical Leads are members of the Kubernetes Organization](#technical-leads-are-members-of-the-kubernetes-organization)\n- [Subproject Leads](#subproject-leads)\n- [Meetings](#meetings)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [X] k/enhancements issue in release milestone and linked to KEP (https://github.com/kubernetes/enhancements/issues/667)\n- [X] KEP approvers have set the KEP status to `implementable`\n- [X] Design details are appropriately documented\n- [X] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [X] Graduation criteria is in place\n- [X] \"Implementation History\" section is up-to-date for milestone\n- [X] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nBuild support for the out-of-tree Azure cloud provider. This involves a well-tested version of the cloud-controller-manager\nthat has feature parity to the kube-controller-manager.\n\n## Motivation\n\nMotivation for supporting out-of-tree providers can be found in the [Cloud Controller Manager KEP](/keps/sig-cloud-provider/20180530-cloud-controller-manager.md).\nThis KEP is specifically tracking progress for the Azure cloud provider.\n\n### Goals\n\n- Develop/test/release the Azure cloud-controller-manager\n- Kubernetes clusters running on Azure should be running the cloud-controller-manager.\n\n### Non-Goals\n\n- Removing in-tree Azure cloud provider code, this effort falls under the [KEP for removing in-tree providers](https://github.com/kubernetes/enhancements/blob/master/keps/sig-cloud-provider/2019-01-25-removing-in-tree-providers.md).\n\n## Proposal\n\nWe propose a set of repositories from the Kubernetes organization to host our cloud provider implementation. Since AzureFile/AzureDisk are also depending on Azure cloud provider, three new projects would be setup:\n\n- [kubernetes/cloud-provider-azure](https://github.com/kubernetes/cloud-provider-azure) would be the main repository for Azure cloud controller manager.\n- [kubernetes-sigs/azuredisk-csi-driver](https://github.com/kubernetes-sigs/azuredisk-csi-driver) would be the repository for AzureDisk CSI plugin.\n- [kubernetes-sigs/azurefile-csi-driver](https://github.com/kubernetes-sigs/azurefile-csi-driver) would be the repository for AzureFile CSI plugin.\n\nThose projects would be subprojects under [SIG Cloud Provider provider-azure](https://github.com/kubernetes/community/tree/master/sig-cloud-provider#provider-azure).\n\n### Documentation\n\nExample manifests, node labels/annotations, service labels/annotations and persistent volumes would be added per [Cloud Provider Documentation KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-cloud-provider/20180731-cloud-provider-docs.md):\n\n```\ncloud-provider-azure/\n└── docs\n    │── example-manifests\n    |   |── in-tree/\n    |   |   ├── apiserver.manifest                 # an example manifest of apiserver using the in-tree integration of this cloud provider\n    |   |   ├── kube-controller-manager.manifest   # an example manifest of kube-controller-manager using the in-tree integration of this cloud provider\n    |   |   └── kubelet.manifest                   # an example manifest of kubelet using the in-tree integration of this cloud provider\n    |   └── out-of-tree/\n    |       ├── apiserver.manifest                 # an example manifest of apiserver using the out-of-tree integration of this cloud provider\n    |       ├── kube-controller-manager.manifest   # an example manifest of kube-controller-manager using the out-of-tree integration of this cloud provider\n    |       ├── cloud-controller-manager.manifest  # an example manifest of cloud-controller-manager using the out-of-tree integration of this cloud provider\n    |       └── kubelet.manifest                   # an example manifest of kubelet using out-of-tree integration of this cloud provider\n    └── resources\n        |── node/\n        |   ├── labels.md        # outlines what annotations that can be used on a Node resource\n        |   ├── annotations.md   # outlines what annotations that can be used on a Node resource\n        |   └── README.md        # outlines any other cloud provider specific details worth mentioning regarding Nodes\n        |── service/\n        |   ├── labels.md        # outlines what annotations that can be used on a Service resource\n        |   ├── annotations.md   # outlines what annotations that can be used on a Service resource\n        |   └── README.md        # outlines any other cloud provider specific details worth mentioning regarding Services\n        └── persistentvolumes/\n            ├── azuredisk\n            |   └── README.md    # outlines CSI drivers of AzureDisk and link to CSI repository\n            └── azurefile\n                └── README.md    # outlines CSI drivers of AzureFile and link to CSI repository\n```\n\n### Implementation Details/Notes/Constraints\n\n- The core of Azure cloud provider would be moved to [kubernetes-sigs/cloud-provider-azure](https://github.com/kubernetes-sigs/cloud-provider-azure).\n- The storage drivers would be moved to [kubernetes-sigs/azuredisk-csi-driver](https://github.com/kubernetes-sigs/azuredisk-csi-driver) and [kubernetes-sigs/azurefile-csi-driver](https://github.com/kubernetes-sigs/azurefile-csi-driver).\n- The credential provider is still under discussion on [kubernetes/cloud-provider#13](https://github.com/kubernetes/cloud-provider/issues/13).\n\n### Risks and Mitigation\n\n#### API throttling\n\nBefore CCM, kubelet supports getting Node information by cloud provider's instance metadata service. This includes:\n\n- NodeName\n- ProviderID\n- NodeAddresses\n- InstanceType\n- AvailabilityZone\n\nBut with CCM, this is not possible anymore because the above functionalities have been moved to cloud controller manager.\n\nSince API throttling is a main issue for large clusters, we have added caches for Azure resources in KCM. Those caches would also be added to CCM. But even with caches, there would still be API throttling issues on cluster provisioning stages and nodes initialization durations would be much longer than KCM because of throttling.\n\nThis issue is being tracked on [kubernetes/cloud-provider#30](https://github.com/kubernetes/cloud-provider/issues/30). Its status would be updated later when it's discussed through sig cloud-provider.\n\n#### Azure credential provider\n\nAzure credential provider is also depending on cloud provider codes. Though Azure Managed Service Identity (MSI) is a way to avoid explicit setting of credentials, MSI is not available on all cases (e.g. MSI may not be authorized to specific ACR repository).\n\nThis issue is being tracked on KEP [Support Instance Metadata Service with Cloud Controller Manager](https://github.com/kubernetes/enhancements/blob/master/keps/sig-cloud-provider/azure/20190722-ccm-instance-metadata.md). It has been marked as implementable and would be implemented in cloud-provider-auzre.\n\n## Design Details\n\n### Test Plan\n\nAzure Cloud Controller provider is reporting conformance test results to TestGrid as per the [Reporting Conformance Test Results to Testgrid KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-cloud-provider/0018-testgrid-conformance-e2e.md).\n\nSee [report](https://testgrid.k8s.io/provider-azure-cloud-provider-azure) for more details.\n\n### Graduation Criteria\n\n- Azure cloud controller manager is moving to GA\n  - Feature compatible with KCM\n  - Conformance tests are passed and published to testgrid\n- CSI drivers for AzureDisk/AzureFile are moving to GA\n  - Feature compatible with KCM\n  - Conformance tests are passed and published to testgrid\n- Azure credential provider is still supported in Kubelet\n  - Feature compatible with KCM\n  - Conformance tests are passed and published to testgrid\n\n### Upgrade / Downgrade Strategy\n\nUpgrade/Downgrade Azure cloud controller manager and CSI drivers together with other master components. The versions for Azure cloud controller manager and CSI drivers should be chosen according to the version skew strategy below.\n\n### Version Skew Strategy\n\nFor each Kubernetes minor releases (e.g. v1.15.x), dedicated Azure cloud controller manager would be released. For CSI drivers, however, they would be released based on [CSI specification versions](https://kubernetes-csi.github.io/docs/).\n\n- The version matrix for Azure cloud controller manager would be documented on [kubernetes/cloud-provider-azure](https://github.com/kubernetes/cloud-provider-azure/blob/master/README.md#current-status).\n- The version matrix for CSI drivers would be documented on [kubernetes-sigs/azuredisk-csi-driver](https://github.com/kubernetes-sigs/azuredisk-csi-driver#container-images--csi-compatibility) and [kubernetes-sigs/azurefile-csi-driver](https://github.com/kubernetes-sigs/azurefile-csi-driver#container-images--csi-compatibility).\n\n## Implementation History\n\nSee [kubernetes/cloud-provider-azure#pulls](https://github.com/kubernetes/cloud-provider-azure/pulls?utf8=%E2%9C%93\u0026q=+is%3Apr+), [kubernetes-sigs/azuredisk-csi-driver#pulls](https://github.com/kubernetes-sigs/azuredisk-csi-driver/pulls?utf8=%E2%9C%93\u0026q=is%3Apr++) and [kubernetes-sigs/azurefile-csi-driver#pulls](https://github.com/kubernetes-sigs/azurefile-csi-driver/pulls?utf8=%E2%9C%93\u0026q=is%3Apr++).\n\n## Technical Leads are members of the Kubernetes Organization\n\nThe Leads run operations and processes governing this subproject.\n\n- @khenidak\n- @feiskyer\n\n## Subproject Leads\n\n- @dstrebel\n- @justaugustus\n\n## Meetings\n\nSig-Azure meetings is expected to have biweekly. SIG Cloud Provider will provide zoom/youtube channels as required. We will have our first meeting after repo has been settled.\n\nMeeting Time: Wednesdays at 09:00 PT (Pacific Time) (biweekly). [Convert to your timezone](http://www.thetimezoneconverter.com/?t=20:00\u0026tz=PT%20%28Pacific%20Time%29).\n\n- [Meeting notes and Agenda](https://docs.google.com/document/d/1SpxvmOgHDhnA72Z0lbhBffrfe9inQxZkU9xqlafOW9k/edit).\n- [Meeting recordings](https://www.youtube.com/watch?v=yQLeUKi_dwg\u0026list=PL69nYSiGNLP2JNdHwB8GxRs2mikK7zyc4).\n"
+  },
+  {
+    "id": "635e5ea4481f816a9d8f26437ea29e3c",
+    "title": "Support Instance Metadata Service with Cloud Controller Manager",
+    "authors": ["@feiskyer"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": ["sig-node"],
+    "reviewers": ["@andrewsykim", "@jagosan"],
+    "approvers": ["@andrewsykim"],
+    "editor": "@feiskyer",
+    "creationDate": "2019-07-22",
+    "lastUpdated": "2019-12-15",
+    "status": "implementable",
+    "seeAlso": ["/keps/sig-cloud-provider/20180530-cloud-controller-manager.md"],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Support Instance Metadata Service with Cloud Controller Manager\n\n## Table of Contents\n\n\u003c!-- TOC --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n- [Alternatives](#alternatives)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n    - [Examples](#examples)\n      - [Removing a deprecated flag](#removing-a-deprecated-flag)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n\u003c!-- /TOC --\u003e\n\n## Release Signoff Checklist\n\n**ACTION REQUIRED:** In order to merge code into a release, there must be an issue in [kubernetes/enhancements] referencing this KEP and targeting a release milestone **before [Enhancement Freeze](https://github.com/kubernetes/sig-release/tree/master/releases)\nof the targeted release**.\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e., [kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be updated for the enhancement to be released.\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nWith [cloud-controller-manager](https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/)(CCM), Kubelet won’t initialize itself from Instance Metadata Service (IMDS). Instead,  CCM would get the node information from cloud APIs. This would introduce more cloud APIs invoking and more possibilities to get throttled, especially for large clusters.\n\nThis proposal aims to add instance metadata service (IMDS) support with CCM. So that, all the nodes could still initialize themselves and reconcile the IP addresses from IMDS.\n\n## Motivation\n\nBefore CCM, kubelet supports getting Node information by the cloud provider's instance metadata service. This includes:\n\n- NodeName\n- ProviderID\n- NodeAddresses\n- InstanceType\n- AvailabilityZone\n\nInstance metadata service could help to reduce API throttling issues and the node's initialization duration. This is especially helpful for large clusters. But with CCM, this is not possible anymore because the above functionality has been moved to CCM.\n\nTake Azure cloud provider for example:\n\n- According to Azure documentation [here](https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-manager-request-limits), for each Azure subscription and tenant, Resource Manager allows up to **12,000 read requests per hour** and 1,200 write requests per hour. That means, on average only 200 read requests could be sent per minute.\n- For different Azure APIs, there’re also additional rate limits based on different durations. For example, there are 3Min and 30Min read limits for VMSS APIs (the numbers below are only for reference since they’re not officially [documented](https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-manager-request-limits)):\n  - Microsoft.Compute/HighCostGetVMScaleSet3Min;200\n  - Microsoft.Compute/HighCostGetVMScaleSet30Min;1000\n  - Microsoft.Compute/VMScaleSetVMViews3Min;5000\n\nBased on those rate limits, getting node’s information for a 5000 cluster may need hours.  Things would be much worse for multiple clusters in the same tenant and subscription.\n\nSo the proposal aims to add IMDS support back with CCM, so that the kubernetes cluster could still be scaled to a large number of nodes.\n\n### Goals\n\n- Allow nodes to be initialized from IMDS.\n- Allow nodes to reconcile the node addresses from IMDS.\n\n### Non-Goals\n\n- Authentication and authorization for each provider implementations.\n- API throttling [issue](https://github.com/kubernetes/kubernetes/issues/60646) on route controller.\n\n## Proposal\n\nSame as kube-controller-manager, the [cloud provider interfaces](https://github.com/kubernetes/cloud-provider/blob/master/cloud.go#L43) could be split into two parts:\n\n- Instance-level interfaces: Instances and Zones\n- Control-plane interfaces, e.g. LoadBalancer and Routes\n\nThe control-plane interfaces are still kept in CCM (who’s name is `cloud-controller-manager`), and deployed on masters. For instance-level interfaces, a new daemonsets would be introduced and implement instance-level interfaces (who’s name is `cloud-node-manager`).\n\nWith these changes, the whole node initialization workflow would be:\n\n- Kubelet specifying `--cloud-provider=external` will add a taint `node.cloudprovider.kubernetes.io/uninitialized` with an effect NoSchedule during initialization.\n- `cloud-node-manager` would initialize the node again with `Instances` and `Zones`.\n- `cloud-controller-manager` then take care of the rest things, e.g. configure the Routes and LoadBalancer for the node.\n\nAfter node initialized, cloud-node-manager would reconcile the IP addresses from IMDS periodically.\n\nConsidering some providers may not require IMDS, cloud-node-manager cloud be enabled optionally by a new option `--enable-node-controller` on cloud-controller-manager. With this new option, there would be three node initialization modes after this proposal:\n\n- 1) Centrally via cloud-controller-manager. All the node initialization, node IP address reconciling and other cloud provider operations are done in CCM.\n  - `cloud-controller-manager --enable-node-controller=true`\n- 2) Using IMDS with cloud-node-manager.\n  - cloud-node-manager running as a daemonset on each node\n  - `cloud-controller-manager enable-node-controller=false`\n- 3) Arbitrary via custom controllers. Customers may also choose their own controllers, which implement the same functions in cloud provider interfaces. The design and deployments are out of this proposal's scope.\n\n## Alternatives\n\nSince there are already a lot of plugins in Kubelet, e.g. CNI, CRI, and CSI. An alternative way is introducing another cloud-provider plugin, e.g. Cloud Provider Interface (CPI).\n\nWhen Kubelet starts, the cloud provider plugin may register itself into Kubelet, and then Kubelet invokes cloud provider plugin to initialize the node.\n\nOne problem is the deployment of those plugins. If daemonsets is used to deploy those cloud provider plugins, then they should be schedulable before kubelet fully initialized the nodes. That means Kubelet may need to initialize itself two times:\n\n- Register the node into Kubernetes without any cloud-specific information.\n- Wait for cloud provider plugins registered and then invoke the plugin to add the cloud-specific information.\n\nThe problem of this way is cloud provider plugin would block node’s initialization, while the plugin itself could be scheduled to that node. Although taint _node.cloudprovider.kubernetes.io/uninitialized_ with an effect NoSchedule could still be applied to solve this issue, separating it to cloud-node-manager would make the whole architecture more clear.\n\n## Design Details\n\n### Test Plan\n\n**Note:** *Section not required until targeted at a release.*\n\nConsider the following in developing a test plan for this enhancement:\n\n- Will there be e2e and integration tests, in addition to unit tests?\n- How will it be tested in isolation vs with other components?\n\nNo need to outline all of the test cases, just the general strategy.\nAnything that would count as tricky in the implementation and anything particularly challenging to test should be called out.\n\nAll code is expected to have adequate tests (eventually with coverage expectations).\nPlease adhere to the [Kubernetes testing guidelines][testing-guidelines] when drafting this test plan.\n\n[testing-guidelines]: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md\n\n### Graduation Criteria\n\n**Note:** *Section not required until targeted at a release.*\n\nDefine graduation milestones.\n\nThese may be defined in terms of API maturity, or as something else. Initial KEP should keep\nthis high-level with a focus on what signals will be looked at to determine graduation.\n\nConsider the following in developing the graduation criteria for this enhancement:\n\n- [Maturity levels (`alpha`, `beta`, `stable`)][maturity-levels]\n- [Deprecation policy][deprecation-policy]\n\nClearly define what graduation means by either linking to the [API doc definition](https://kubernetes.io/docs/concepts/overview/kubernetes-api/#api-versioning),\nor by redefining what graduation means.\n\nIn general, we try to use the same stages (alpha, beta, GA), regardless how the functionality is accessed.\n\n[maturity-levels]: https://git.k8s.io/community/contributors/devel/sig-architecture/api_changes.md#alpha-beta-and-stable-versions\n[deprecation-policy]: https://kubernetes.io/docs/reference/using-api/deprecation-policy/\n\n#### Examples\n\nThese are generalized examples to consider, in addition to the aforementioned [maturity levels][maturity-levels].\n\n##### Alpha -\u003e Beta Graduation\n\n- Gather feedback from developers and surveys\n- Complete features A, B, C\n- Tests are in Testgrid and linked in KEP\n\n##### Beta -\u003e GA Graduation\n\n- N examples of real world usage\n- N installs\n- More rigorous forms of testing e.g., downgrade tests and scalability tests\n- Allowing time for feedback\n\n**Note:** Generally we also wait at least 2 releases between beta and GA/stable, since there's no opportunity for user feedback, or even bug reports, in back-to-back releases.\n\n##### Removing a deprecated flag\n\n- Announce deprecation and support policy of the existing flag\n- Two versions passed since introducing the functionality which deprecates the flag (to address version skew)\n- Address feedback on usage/changed behavior, provided on GitHub issues\n- Deprecate the flag\n\n**For non-optional features moving to GA, the graduation criteria must include [conformance tests].**\n\n[conformance tests]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/conformance-tests.md\n\n### Upgrade / Downgrade Strategy\n\nIf applicable, how will the component be upgraded and downgraded? Make sure this is in the test plan.\n\nConsider the following in developing an upgrade/downgrade strategy for this enhancement:\n\n- What changes (in invocations, configurations, API use, etc.) is an existing cluster required to make on upgrade in order to keep previous behavior?\n- What changes (in invocations, configurations, API use, etc.) is an existing cluster required to make on upgrade in order to make use of the enhancement?\n\n### Version Skew Strategy\n\nIf applicable, how will the component handle version skew with other components? What are the guarantees? Make sure\nthis is in the test plan.\n\nConsider the following in developing a version skew strategy for this enhancement:\n\n- Does this enhancement involve coordinating behavior in the control plane and in the kubelet? How does an n-2 kubelet without this feature available behave when this feature is used?\n- Will any other components on the node change? For example, changes to CSI, CRI or CNI may require updating that component before the kubelet.\n\n## Implementation History\n\nMajor milestones in the life cycle of a KEP should be tracked in `Implementation History`.\nMajor milestones might include\n\n- the `Summary` and `Motivation` sections being merged signaling SIG acceptance\n- the `Proposal` section being merged signaling agreement on a proposed design\n- the date implementation started\n- the first Kubernetes release where an initial version of the KEP was available\n- the version of Kubernetes where the KEP graduated to general availability\n- when the KEP was retired or superseded\n"
+  },
+  {
+    "id": "690b0290980b89c6a2188d7d5047f1dd",
+    "title": "Cloud Provider for Alibaba Cloud",
+    "authors": ["@aoxn"],
+    "owningSig": "sig-cloud-provider",
+    "participatingSigs": null,
+    "reviewers": ["@andrewsykim"],
+    "approvers": ["@andrewsykim", "@hogepodge", "@jagosan"],
+    "editor": "TBD",
+    "creationDate": "2018-06-20",
+    "lastUpdated": "2018-06-20",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Cloud Provider for Alibaba Cloud\n\nThis is a KEP for adding ```Cloud Provider for Alibaba Cloud``` into the Kubernetes ecosystem.\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Prerequisites](#prerequisites)\n  - [Repository Requirements](#repository-requirements)\n  - [User Experience Reports](#user-experience-reports)\n  - [Testgrid Integration](#testgrid-integration)\n  - [CNCF Certified Kubernetes](#cncf-certified-kubernetes)\n  - [Documentation](#documentation)\n  - [Technical Leads are members of the Kubernetes Organization](#technical-leads-are-members-of-the-kubernetes-organization)\n- [Proposal](#proposal)\n  - [Repositories](#repositories)\n  - [Meetings](#meetings)\n  - [Others](#others)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nAlibaba Cloud provides the Cloud Provider interface implementation as an out-of-tree cloud-controller-manager. It allows Kubernetes clusters to leverage the infrastructure services of Alibaba Cloud .\nIt is original open sourced project is [https://github.com/AliyunContainerService/alicloud-controller-manager](https://github.com/AliyunContainerService/alicloud-controller-manager)\n\n## Motivation\n\n### Goals\n\nCloud Provider of Alibaba Cloud  implements interoperability between Kubernetes cluster and Alibaba Cloud. In this project, we will dedicated in:\n- Provide reliable, secure and optimized integration with Alibaba Cloud for Kubernetes\n\n- Help on the improvement for decoupling cloud provider specifics from Kubernetes implementation.\n\n\n\n### Non-Goals\n\nThe networking and storage support of Alibaba Cloud for Kubernetes will be provided by other projects.\n\nE.g.\n\n* [Flannel network for Alibaba Cloud VPC](https://github.com/coreos/flannel)\n* [FlexVolume for Alibaba Cloud](https://github.com/AliyunContainerService/flexvolume)\n\n\n## Prerequisites\n\n1. The VPC network is supported in this project. The support for classic network or none ECS environment will be out-of-scope.\n2. When using the instance profile for authentication, an instance role is required to attach to the ECS instance firstly.\n3. Kubernetes version v1.7 or higher\n\n### Repository Requirements\n\n[Alibaba Cloud Controller Manager](https://github.com/AliyunContainerService/alicloud-controller-manager) is a working implementation of the [Kubernetes Cloud Controller Manager](https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/).\n\nThe repo requirements is mainly a copy from [cloudprovider KEP](https://github.com/kubernetes/community/blob/master/keps/sig-cloud-provider/0002-cloud-controller-manager.md#repository-requirements). Open the link for more detail.\n\n### User Experience Reports\nAs a CNCF Platinum member, Alibaba Cloud is dedicated in providing users with highly secure , stable and efficient cloud service.\nUsage of aliyun container services can be seen from github issues in the existing alicloud controller manager repo: https://github.com/AliyunContainerService/alicloud-controller-manager/issues\n\n### Testgrid Integration\n Alibaba cloud provider is reporting conformance test results to TestGrid as per the [Reporting Conformance Test Results to Testgrid KEP](https://github.com/kubernetes/community/blob/master/keps/sig-cloud-provider/0018-testgrid-conformance-e2e.md).\n See [report](https://k8s-testgrid.appspot.com/conformance-alibaba-cloud-provider#Alibaba%20Cloud%20Provider,%20v1.10) for more details.\n\n### CNCF Certified Kubernetes\n Alibaba cloud provider is accepted as part of the [Certified Kubernetes Conformance Program](https://github.com/cncf/k8s-conformance).\n For v1.11 See [https://github.com/cncf/k8s-conformance/tree/master/v1.11/alicloud](https://github.com/cncf/k8s-conformance/tree/master/v1.11/alicloud)\n For v1.10 See [https://github.com/cncf/k8s-conformance/tree/master/v1.10/alicloud](https://github.com/cncf/k8s-conformance/tree/master/v1.10/alicloud)\n For v1.9 See [https://github.com/cncf/k8s-conformance/tree/master/v1.9/alicloud](https://github.com/cncf/k8s-conformance/tree/master/v1.9/alicloud)\n For v1.8 See [https://github.com/cncf/k8s-conformance/tree/master/v1.8/alicloud](https://github.com/cncf/k8s-conformance/tree/master/v1.8/alicloud)\n\n### Documentation\n \n Alibaba CloudProvider provide users with multiple documentation on build \u0026 deploy \u0026 utilize CCM. Please refer to [https://github.com/AliyunContainerService/alicloud-controller-manager/tree/master/docs](https://github.com/AliyunContainerService/alicloud-controller-manager/tree/master/docs) for more details.\n \n### Technical Leads are members of the Kubernetes Organization\n\nThe Leads run operations and processes governing this subproject.\n\n-  @cheyang Special Tech Leader, Alibaba Cloud. Kubernetes Member\n\n## Proposal\n\nHere we propose a repository from Kubernetes organization to host our cloud provider implementation.  Cloud Provider of Alibaba Cloud would be a subproject under Kubernetes community.\n\n### Repositories\n\nCloud Provider of Alibaba Cloud will need a repository under Kubernetes org named ```kubernetes/cloud-provider-alibaba-cloud``` to host any cloud specific code.\nThe initial owners will be indicated in the initial OWNER files.\n\nAdditionally, SIG-cloud-provider take the ownership of the repo but Alibaba Cloud should have the fully autonomy permission to operator on this subproject.\n\n### Meetings\n\nCloud Provider meetings is expected to have biweekly. SIG Cloud Provider will provide zoom/youtube channels as required. We will have our first meeting after repo has been settled.\n\nRecommended Meeting Time: Wednesdays at 20:00 PT (Pacific Time) (biweekly). [Convert to your timezone](http://www.thetimezoneconverter.com/?t=20:00\u0026tz=PT%20%28Pacific%20Time%29).\n- Meeting notes and Agenda.\n- Meeting recordings.\n\n\n### Others\n"
+  },
+  {
+    "id": "28357c27e8ff1d638f93a6696301b0f5",
+    "title": "Addons via Operators",
+    "authors": ["@justinsb"],
+    "owningSig": "sig-cluster-lifecycle",
+    "participatingSigs": null,
+    "reviewers": ["@luxas", "@roberthbailey", "@timothysc"],
+    "approvers": ["@timothysc"],
+    "editor": "TBD",
+    "creationDate": "2019-01-28",
+    "lastUpdated": "2019-03-11",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Addons via Operators\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Success Criteria](#success-criteria)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Infrastructure Needed](#infrastructure-needed)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nWe propose to use operators for managing cluster addons.  Each addon will have\nits own CRD, and users will be able to perform limited tailoring of the addon\n(install/don’t install, choose version, primary feature selection) by modifying\nthe instance of the CRD.  The operator encodes any special logic (e.g. dependencies) needed to\ninstall the addon.\n\nBy creating a CRD per addon, we are able make use of the kubernetes API\nmachinery for those per-addon options.\n\nWe will create tooling to make it easy to build addon operators that follow the\nbest practices we identify as part of this work.  For example, we expect that\nmost addons will be declarative, and likely be specified as part of a \"cluster\nbundle\", so we will make it easy to build basic addon operators that follow\nthese patterns.\n\nWe hope that components will choose to maintain their own operators, encoding\ntheir knowledge of how best to operate their addon.\n\n\n## Motivation\n\nAddons are components that are managed alongside the lifecycle of the cluster.\nThey are often tied to or dependent on the configuration of other cluster\ncomponents.  Management of these components has proved complicated.  Our\nexisting solution in the form of the bash addon-manager has many known\nshortcomings and is not widely adopted.  As we focus more development outside of\nthe kubernetes/kubernetes repo, we expect more addon components of greater\ncomplexity.  This is one of the long-standing backlog items for\nsig-cluster-lifecycle.\n\nUse of operators is now generally accepted, and the benefits to other\napplications are generally recognized.  We aim to bring the benefits of\noperators to addons also.\n\n### Goals\n\n* Explore the use of operators for managing addons\n* Create patterns, libraries \u0026 tooling so that addons are of high quality,\n  consistent in their API surface (common fields on CRDs, use of Application\n  CRD, consistent labeling of created resources), yet are easy to build.\n* Build addons for the basic set of components, acting as a quality reference\n  implementation suitable for production use.  We aim also to demonstrate the\n  utility and explore any challenges, and to verify that the tooling does make\n  addon-development easy.\n\n\n### Non-Goals\n\n* We do not intend to mandate that all installation tools use addon operators;\n  installation tools are free to choose their own path.\n* Management of non-addons is out of scope (for example installation of end-user\n  applications, or of packaged software that is not an addon)\n\n\n## Proposal\n\nThis is the current plan of action; it is based on experience gathered and work\ndone for Google’s GKE-on-prem product.  However we don’t expect this will\nnecessarily be directly applicable in the OSS world and we are open to change as\nwe discover new requirements.\n\n* Extend kubebuilder \u0026 controller-runtime to make it easy to build operators for\n  addons\n* Build addons for the primary addons currently in the cluster/ directory, at\n  least including those required to bring up a conformant cluster.  Proposed\n  list: CoreDNS, kube-proxy, dashboard, metrics-server, localdns-agent.\n* Plug in those addons operators into kube-up / cluster-api / kubeadm / kops /\n  others (subject to those projects being interested)\n* Develop at least one addon operator outside of kubernetes/kubernetes\n  (LocalDNS-Cache?) and figure out how it can be used despite being out-of-tree\n* Investigate use of webhooks to prevent accidental mutation of child objects\n* Investigate the RBAC story for addons - currently the operator must itself\n  have all the permissions that the addon needs, which is not really\n  least-privilege.  But it is not clear how to side-step this, nor that any of\n  the alternatives would be better or more secure.\n* Investigate use of patching mechanisms (as seen in `kubectl patch` and\n  `kustomize`) to support advanced tailoring of addons.  The goal here is to\n  make sure that everyone can use the addon operators, even if they “love it but\n  just need to change one thing”.  This ensures that the addon operators\n  themselves can remain bounded in scope and complexity.  Patching will fail if\n  the underlying addon changes dramatically, so we'll likely have a \"patch\n  incompatible with new version\" error - and generally addons should avoid\n  gratuitously changing their structure.\n* We should develop a convention (labels / owner-refs) so that we are able to\n  discover which CRDs are cluster-addons, and there is no confusion with\n  application operators.\n\nWe expect the following functionality to be common to all operators for addons:\n\n* A CRD per addon\n* Common fields in spec that define the version and/or channel\n* Common fields in status that expose the current health \u0026 version information\n  of the addon\n* Addons follow a common structure, with an instance of the CRD as root object,\n  an Application CRD instance, consistent labels of all objects\n* Some form of protection or rapid reconciliation to prevent accidental\n  modification of child objects\n* Operators are declaratively driven, and can source manifests via https\n  (including mirrors), or from data stored in the cluster itself\n  (e.g. configmaps or cluster-bundle CRD, useful for airgapped)\n* Operators are able to expose different update behaviours: automatic immediate\n  updates; notification of update-available in status; purely manual updates\n* Operators are able to observe other CRDs to perform basic sequencing\n* Addon manifests are able express an operator minimum version requirement, so\n  that an addon with new requirements can require that the operator be updated\n  first\n* Airgapped operation should be possible by combining a registry mirror and\n  storage of the underlying manifest in the cluster itself.\n\n\nAn example can make this easier to understand, here is what a CRD instance for\nkube-proxy might look like:\n\n```yaml\napiVersion: addons.sigs.k8s.io/v1alpha1\nkind: KubeProxy\nmetadata:\n  name: default\n  namespace: kube-system\nspec:\n  clusterCidr: 100.64.0.0/10\n  version: 1.14.4\nstatus:\n  healthy: true\n```\n\nThis particular manifest is pinned to `version: 1.14.4`.  We could also\nsubscribe to a stream of updates with a field like `channel: stable`.\n\n### Risks and Mitigations\n\nThis will involve running a large number of new controllers.  This will require\nmore resources; we can mitigate this by combining them into a single binary\n(similar to kube-controller-manager).\n\nAutomatically updating addons could result in new SPOFs, we can mitigate this\nthrough mirroring (including support for air-gapped mirrors).\n\nProviding a good set of addons could result in a monoculture where mistakes\naffect most/all kubernetes clusters (even if we don’t mandate adoption, if we\nsucceed we hope for widespread adoption).  We can continue with our strategies\nthat we use for core components such as kube-apiserver: primarily we must keep\nthe notion of stable vs less-stable releases, to stagger the risk of a bad\nrollout.  We must also consider this a trade-off against the risk that without\ncoordination each piece of tooling must reinvent the wheel; we expect more\nmistakes (even measured per cluster) in that scenario.\n\nTest and release may become more complicated because of fragmentation across\nrepos.  Mitigation: be disciplined about versioning of operators and addons and\nencourage installation tooling to pin to a particular version of both for a\nparticular release.  We need to set up automated builds (with CI) for rapid\nreleases so installation tooling is not blocked waiting for operator releases.\nWe need to set up a mechanism so that addons can be updated without requiring an\noperator update.  With this, if tooling is able to pin to particular addon\nversions, that should be at parity with the \"embedded manifest\" approach that is\nwidely used currently.  (We hope to enable usage that is less lock-step, but\nthat itself will likely require new approaches for testing and release)\n\n## Success Criteria\n\nWe will succeed if addon operators are:\n\n* Used: addon operators are adopted by the majority of cluster installation\ntooling\n* Useful: users are generally satisfied with the functionality of addon\noperators and are not trying to work around them, or making lots of proposals /\nPRs to extend them\n* Ubiquitous: the majority of components include an operator\n* Federated: the components maintain their own operators, encoding their\nknowledge of how best to run their addon.\n\n## Graduation Criteria\n\nalpha: addon-operators are used to manage kube-proxy \u0026 CoreDNS:\n* in kube-up\n* in kubeadm (at least as an option)\n* in kops\n* in cluster-api-provider-aws \u0026 cluster-api-provider-gcp\n* adoption is documented for use by other tooling / self-managed clusters\n\n(post-alpha criteria will be added post-alpha)\n\n## Implementation History\n\n[Addon Operator session](https://www.youtube.com/watch?v=LPejvfBR5_w) given by jrjohnson \u0026 justinsb at Kubecon NA - Dec 2018\nKEP created - Jan 29 2019\n\n## Infrastructure Needed\n\nInitial development of the tooling can probably take place as part of\nkubebuilder\n\nWe should likely create a repo for holding the operators themselves.  Eventually\nwe would hope these would migrate to the various addon components, so we could\nalso just store these under e.g. cluster-api.\n\nWe are requesting to be a subproject under sig-cluster-lifecycle.\n"
+  },
+  {
+    "id": "27d14189d34067706d3b3cafcc6d2c5d",
+    "title": "Kubernetes Cluster Management API",
+    "authors": ["@roberthbailey", "@pipejakob"],
+    "owningSig": "sig-cluster-lifecycle",
+    "participatingSigs": null,
+    "reviewers": ["@justinsb", "@timothysc"],
+    "approvers": ["@justinsb", "@timothysc"],
+    "editor": "@justinsb",
+    "creationDate": "2018-01-19",
+    "lastUpdated": "2019-04-04",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Kubernetes Cluster Management API\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-goals](#non-goals)\n  - [Challenges and Open Questions](#challenges-and-open-questions)\n- [Proposal](#proposal)\n  - [Driving Use Cases](#driving-use-cases)\n  - [Cluster-level API](#cluster-level-api)\n  - [Machine API](#machine-api)\n    - [Capabilities](#capabilities)\n    - [Overview](#overview)\n    - [In-place vs. Replace](#in-place-vs-replace)\n    - [Omitted Capabilities](#omitted-capabilities)\n    - [Conditions](#conditions)\n    - [Types](#types)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Drawbacks](#drawbacks)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nWe are building a set of Kubernetes cluster management APIs to enable common cluster lifecycle operations (install, upgrade, repair, delete) across disparate environments.\nWe represent nodes and other infrastructure in Kubernetes-style APIs to enable higher level controllers to update the desired state of the cluster (e.g. the autoscaling controller requesting additional machines) and reconcile the world with that state (e.g. communicating with cloud providers to create or delete virtual machines).\nWith the full state of the cluster represented as API objects, Kubernetes installers can use them as a common configuration language, and more sophisticated tooling can be built in an environment-agnostic way.\n\n## Motivation\n\nKubernetes has a common set of APIs (see the [Kubernetes API Conventions](https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md)) to orchestrate containers regardless of deployment mechanism or cloud provider.\nKubernetes also has APIs for handling some infrastructure, like load-balancers, ingress rules, or persistent volumes, but not for creating new machines.\nAs a result, the deployment mechanisms that manage Kubernetes clusters each have unique APIs and implementations for how to handle lifecycle events like cluster creation or deletion, master upgrades, and node upgrades.\nAdditionally, the cluster-autoscaler is responsible not only for determining when the cluster should be scaled, but also responsible for adding capacity to the cluster by interacting directly with the cloud provider to perform the scaling.\nWhen another component needs to create or destroy virtual machines, like the node auto provisioner, it would similarly need to reimplement the logic for interacting with the supported cloud providers (or reuse the same code to prevent duplication).\n\n### Goals\n\n* The cluster management APIs should be declarative, Kubernetes-style APIs that follow our existing [API Conventions](https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md).\n* To the extent possible, we should separate state that is environment-specific from environment-agnostic.\n   * However, we still want the design to be able to utilize environment-specific functionality, or else it likely won’t gain traction in favor of other tooling that is more powerful.\n\n### Non-goals\n\n* To add these cluster management APIs to Kubernetes core.\n* To support infrastructure that is irrelevant to Kubernetes clusters.\n   * We are not aiming to create terraform-like capabilities of creating any arbitrary cloud resources, nor are we interested in supporting infrastructure used solely by applications deployed on Kubernetes. The goal is to support the infrastructure necessary for the cluster itself.\n* To convince every Kubernetes lifecycle product ([kops](https://github.com/kubernetes/kops), [kubespray](https://github.com/kubernetes-incubator/kubespray), [Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine/), [Azure Kubernetes Service](https://azure.microsoft.com/en-us/services/kubernetes-service/), [Elastic Container Service for Kubernetes](https://aws.amazon.com/eks/), etc.) to support these APIs.\n   * There is value in having consistency between installers and broad support for the cluster management APIs and in having common infrastructure reconcilers used post-installation, but 100% adoption isn't an immediate goal.\n* To model state that is purely internal to a deployer.\n   * Many Kubernetes deployment tools have intermediate representations of resources and other internal state to keep track of. They should continue to use their existing methods to track internal state, rather than attempting to model it in these APIs.\n\n### Challenges and Open Questions\n\n* Should a single Kubernetes cluster only house definitions for itself?\n   * If so, that removes the ability to have a single cluster control the reconciliation of infrastructure for other clusters.\n   * However, with the concurrent [Cluster Registry](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/multicluster/cluster-registry/api-design.md) project, a good separation of responsibilities would be that the Cluster Registry API is responsible for indexing multiple clusters, each of which would only have to know about itself. In order to achieve cross-cluster reconciliation, a controller would need to integrate with a Cluster Registry for discovery.\n* Should a cluster’s control plane definition should be housed within that same cluster.\n   * If the control plane becomes unhealthy, then it won’t be able to rectify itself without external intervention. If the control plane configuration lives elsewhere, and the controllers reconciling its state are able to act in the face of control plane failure, then this API could be used to fix a misconfigured control plane that is unresponsive.\n* Should our representation of Nodes allow declarative versioning of non-Kubernetes packages, like the container runtime, the Linux kernel, etc.?\n   * It potentially enables the use case of smaller, in-place upgrades to nodes without changing the node image.\n   * We may be able to leverage cloud-init to some extent, but since it isn’t supported across all cloud/distributions, and doesn’t support upgrades (or any actions beyond initialization), this may devolve into rolling our own solution.\n* Should the Cluster API bother with control plane configuration, or expect each component to use component config?\n   * One option is to allow arbitrary API objects to be defined during cluster initialization, which will be a combination of Cluster objects, NodeSet objects, and ConfigMaps for relevant component config. This makes the Cluster API less comprehensive, but avoids redundancy and more accurately reflects the desired state of the cluster.\n   * Another option is to have key component config embedded in the Cluster API, which will then be created as the appropriate ConfigMaps during creation. This would be used as a convenience during cluster creation, and then the separate ConfigMaps become the authoritative configuration, potentially with a control loop to propagate changes from the embedded component config in the Cluster API to the appropriate (authoritative) ConfigMaps on an ongoing basis.\n* Do we want to allow for arbitrary node boot scripts?\n   * Some existing tools like kubicorn support this, but the user demand isn’t clear yet.\n   * Also see https://github.com/kubernetes/kops/issues/387\n      * Kops now has hooks\n* Are there any environments in which it only makes sense to refer to a group of homogeneous nodes, instead of individual ones?\n   * The current proposal is to start with individual objects to represent each declarative node (called a “Machine”), which allows us to build support for Sets and Deployments on top of them in the future. However, does this simplification break for any environment we want to support?\n\n\n## Proposal\n\n### Driving Use Cases\n\n_TODO_: Separate out the use cases that are focused on the control plane vs. those focused on nodes.\n\n\nThese use cases are in scope for our v1alpha1 API design and initial prototype implementation:\n\n* Initial cluster creation using these API objects in yaml files (implemented via client-side bootstrapping of resources)\n   * Rather than each Kubernetes installer having its own custom APIs and cluster definitions, they could be fed the definition of the cluster via serialized API objects. This would lower the friction of moving between different lifecycle products.\n* Declarative Kubernetes upgrades for the control plane and kubelets\n* Declarative upgrades for node OS images\n* Maintaining consistency of control plane and machine configuration across different clusters / clouds\n   * By representing important cluster configuration via declarative objects, operations like “diffing” the configuration of two clusters becomes very straightforward. Also, reconcilers can be written to ensure that important cluster configuration is kept in sync between different clusters by simply copying objects.\n* Cloud adoption / lift and shift / liberation\n\nThese use cases are in scope for the project, but post-v1alpha1:\n\n* Server-side node draining\n* Autoscaling\n   * Currently, the OSS cluster autoscaler has the responsibility of determining the right size of the cluster and calling the cloud provider to perform the scaling (supporting every cloud provider directly). Modeling groups of nodes in a declarative way would allow autoscalers to only need to worry about the correct cluster size and error handling when that can’t be achieved (e.g. in the case of stockouts), and then separate cloud controllers can be responsible for creating and deleting nodes to reconcile that state and report any errors encountered.\n* Integration with the Cluster Registry API\n   * Automatically add a new cluster to a registry, support tooling that works across multiple clusters using a registry, delete a cluster from a registry.\n* Supporting other common tooling, like monitoring\n\nThese use cases are out of scope entirely:\n\n* Creating arbitrary cloud resources\n\n### Cluster-level API\n\nThis level of the Cluster Management API describes the global configuration of a cluster. It should be capable of representing the versioning and configuration of the entire control plane, irrespective of the representation of nodes.\n\nGiven the recent efforts of SIG Cluster Lifecycle to make kubeadm the de facto standard toolkit for cloud- and vendor-agnostic cluster initialization, and because kubeadm has [an existing API](https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/apis/kubeadm/v1alpha3/types.go) to define the global configuration for a cluster, it makes sense to coalesce the global portion of the Cluster API with the API used by “kubeadm init” to configure a cluster master.\n\nA current goal is to make these APIs as cloud-agnostic as possible, so that the entire definition of a Cluster could remain reasonably in-sync across different deployments potentially in different cloud providers, which would help enable hybrid usecases where it’s desirable to have key configuration stay in sync across different clusters potentially in different clouds/environments. However, this goal is balanced against making the APIs coherent and usable, which strict separation may harm.\n\nThe full types for this API can be seen and were initially discussed in [kube-deploy#306](https://github.com/kubernetes/kube-deploy/pull/306).\n\n### Machine API\n\n#### Capabilities\n\nThe set of node capabilities that this proposal is targeting for v1alpha1 are:\n1. A new Node can be created in a declarative way, including Kubernetes version and container runtime version. It should also be able to specify provider-specific information such as OS image, instance type, disk configuration, etc., though this will not be portable.\n1. A specific Node can be deleted, freeing external resources associated with it.\n1. A specific Node can have its kubelet version upgraded or downgraded in a declarative way\\*.\n1. A specific Node can have its container runtime changed, or its version upgraded or downgraded, in a declarative way\\*.\n1. A specific Node can have its OS image upgraded or downgraded in a declarative way\\*.\n\n\\* It is an implementation detail of the provider if these operations are performed in-place or via Node replacement.\n\n#### Overview\n\nThis proposal introduces a new API type: **Machine**.\n\nA \"Machine\" is the declarative spec for a Node, as represented in Kubernetes core. If a new Machine object is created, a provider-specific controller will handle provisioning and installing a new host to register as a new Node matching the Machine spec. If the Machine's spec is updated, a provider-specific controller is responsible for updating the Node in-place or replacing the host with a new one matching the updated spec. If a Machine object is deleted, the corresponding Node should have its external resources released by the provider-specific controller, and should be deleted as well.\n\nFields like the kubelet version, the container runtime to use, and its version, are modeled as fields on the Machine's spec. Any other information that is provider-specific, though, is part of an opaque ProviderConfig string that is not portable between different providers.\n\nThe ProviderConfig is recommended to be a serialized API object in a format owned by that provider, akin to the [Component Config](https://docs.google.com/document/d/1arP4T9Qkp2SovlJZ_y790sBeiWXDO6SG10pZ_UUU-Lc/edit) pattern. This will allow the configuration to be strongly typed, versioned, and have as much nested depth as appropriate. These provider-specific API definitions are meant to live outside of the Machines API, which will allow them to evolve independently of it. Attributes like instance type, which network to use, and the OS image all belong in the ProviderConfig.\n\n#### In-place vs. Replace\n\nOne simplification that might be controversial in this proposal is the lack of API control over \"in-place\" versus \"replace\" reconciliation strategies. For instance, if a Machine's spec is updated with a different version of kubelet than is actually running, it is up to the provider-specific controller whether the request would best be fulfilled by performing an in-place upgrade on the Node, or by deleting the Node and creating a new one in its place (or reporting an error if this particular update is not supported). One can force a Node replacement by deleting and recreating the Machine object rather than updating it, but no similar mechanism exists to force an in-place change.\n\nAnother approach considered was that modifying an existing Machine should only ever attempt an in-place modification to the Node, and Node replacement should only occur by deleting and creating a new Machine. In that case, a provider would set an error field in the status if it wasn't able to fulfill the requested in-place change (such as changing the OS image or instance type in a cloud provider).\n\nThe reason this approach wasn't used was because most cluster upgrade tools built on top of the Machines API would follow the same pattern:\n\n```\nfor machine in machines:\n    attempt to upgrade machine in-place\n    if error:\n        create new machine\n        delete old machine\n```\n\nSince updating a Node in-place is likely going to be faster than completely replacing it, most tools would opt to use this pattern to attempt an in-place modification first, before falling back to a full replacement.\n\nIt seems like a much more powerful concept to allow every tool to instead say:\n\n```\nfor machine in machines:\n    update machine\n```\n\nand allow the provider to decide if it is capable of performing an in-place update, or if a full Node replacement is necessary.\n\n#### Omitted Capabilities\n\n**A scalable representation of a group of nodes**\n\nGiven the existing targeted capabilities, this functionality could easily be built client-side via label selectors to find groups of Nodes and using (1) and (2) to add or delete instances to simulate this scaling.\n\nIt is natural to extend this API in the future to introduce the concepts of MachineSets and MachineDeployments that mirror ReplicaSets and Deployments, but an initial goal is to first solidify the definition and behavior of a single Machine, similar to how Kubernetes first solidifed Pods.\n\nA nice property of this proposal is that if provider controllers are written solely against Machines, the concept of MachineSets can be implemented in a provider-agnostic way with a generic controller that uses the MachineSet template to create and delete Machine instances. All Machine-based provider controllers will continue to work, and will get full MachineSet functionality for free without modification. Similarly, a MachineDeployment controller could then be introduced to generically operate on MachineSets without having to know about Machines or providers. Provider-specific controllers that are actually responsible for creating and deleting hosts would only ever have to worry about individual Machine objects, unless they explicitly opt into watching higher-level APIs like MachineSets in order to take advantage of provider-specific features like AutoScalingGroups or Managed Instance Groups.\n\nHowever, this leaves the barrier to entry very low for adding new providers: simply implement creation and deletion of individual Nodes, and get Sets and Deployments for free.\n\n**A provider-agnostic mechanism to request new nodes**\n\nIn this proposal, only certain attributes of Machines are provider-agnostic and can be operated on in a generic way. In other iterations of similar proposals, much care had been taken to allow the creation of truly provider-agnostic Machines that could be mapped to provider-specific attributes in order to better support usecases around automated Machine scaling. This introduced a lot of upfront complexity in the API proposals.\n\nThis proposal starts much more minimalistic, but doesn't preclude the option of extending the API to support these advanced concepts in the future.\n\n**Dynamic API endpoint**\n\nThis proposal lacks the ability to declaratively update the kube-apiserver endpoint for the kubelet to register with. This feature could be added later, but doesn't seem to have demand now. Rather than modeling the kube-apiserver endpoint in the Machine object, it is expected that the cluster installation tool resolves the correct endpoint to use, starts a provider-specific Machines controller configured with this endpoint, and that the controller injects the endpoint into any hosts it provisions.\n\n#### Conditions\n\n[bgrant0607](https://github.com/bgrant0607) and [erictune](https://github.com/erictune) have indicated that the API pattern of having \"Conditions\" lists in object statuses is soon to be deprecated. These have generally been used as a timeline of state transitions for the object's reconciliation, and difficult to consume for clients that just want a meaningful representation of the object's current state. There are no existing examples of the new pattern to follow instead, just the guidance that we should use top-level fields in the status to represent meaningful information. We can revisit the specifics when new patterns start to emerge in core.\n\n#### Types\n\nThe full Machine API types can be found and discussed in [kube-deploy#298](https://github.com/kubernetes/kube-deploy/pull/298).\n\n## Graduation Criteria\n\n__TODO__\n\n## Implementation History\n\n* **December 2017 (KubeCon Austin)**: Prototype implementation on Google Compute Engine using Custom Resource Definitions\n\n## Drawbacks\n\n__TODO__\n\n## Alternatives\n\n__TODO__\n"
+  },
+  {
+    "id": "9ba0ece1f013e6701609445fe3f60834",
+    "title": "etcdadm",
+    "authors": ["@justinsb"],
+    "owningSig": "sig-cluster-lifecycle",
+    "participatingSigs": null,
+    "reviewers": ["@roberthbailey", "@timothysc"],
+    "approvers": ["@roberthbailey", "@timothysc"],
+    "editor": "TBD",
+    "creationDate": "2018-10-22",
+    "lastUpdated": "2018-10-22",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# etcdadm - automation for etcd clusters\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Manual Cluster Creation](#manual-cluster-creation)\n    - [Automatic Cluster Creation](#automatic-cluster-creation)\n    - [Automatic Cluster Creation with EBS volumes](#automatic-cluster-creation-with-ebs-volumes)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Infrastructure Needed](#infrastructure-needed)\n\u003c!-- /toc --\u003e\n\n## Summary\n\netcdadm makes operation of etcd for the Kubernetes control plane easy, on clouds\nand on bare-metal, including both single-node and HA configurations.\n\nIt is able to perform cluster reconfigurations, upgrades / downgrades, and\nbackups / restores.\n\n## Motivation\n\nToday each installation tool must reimplement etcd operation, and this is\ndifficult.  It also leads to ecosystem fragmentation - e.g. etcd backups from\none tool are not necessarily compatible with the backups from other tools.  The\nfailure modes are subtle and rare, and thus the kubernetes project benefits from\nhaving more collaboration.\n\n\n### Goals\n\nThe following key tasks are in scope:\n\n* Cluster creation\n* Cluster teardown\n* Cluster resizing / membership changes\n* Cluster backups\n* Disaster recovery or restore from backup\n* Cluster upgrades\n* Cluster downgrades\n* PKI management\n\nWe will implement this functionality both as a base layer of imperative (manual\nCLI) operation, and a self-management layer which should enable automated\nin \"safe\" scenarios (with fallback to manual operation).\n\nWe'll also optionally support limited interaction with cloud infrastructure, for\nexample for mounting volumes and peer-discovery.  This is primarily for the\nself-management layer, but we'll expose it via etcdadm for consistency and for\npower-users.  The tasks are limited today to listing \u0026 mounting a persistent\nvolume, and listing instances to find peers.  A full solution for management of\nmachines or networks (for example) is out of scope, though we might share some\nexample configurations for exposition.  We expect kubernetes installation\ntooling to configure the majority of the cloud infrastructure here, because both\nthe configurations and the configuration tooling varies widely.\n\nThe big reason that volume mounting is in scope is that volume mounting acts as\na simple mutex on most clouds - it is a cheap way to boost the safety of our\nleader/gossip algorithms, because we have an external source of truth.\n\nWe'll also support reading \u0026 writing backups to S3 / GCS etc.\n\n### Non-Goals\n\n* The project is not targeted at operation of an etcd cluster for use other than\n  by Kubernetes apiserver.  We are not building a general-purpose etcd operation\n  toolkit.  Likely it will work well for other use-cases, but other tools may be\n  more suitable.\n* As described above, we aren't building a full \"turn up an etcd cluster on a\n  cloud solution\"; we expect this to be a building block for use by kubernetes\n  installation tooling (e.g. cluster API solutions).\n\n## Proposal\n\nWe will combine the [etcdadm](https://github.com/platform9/etcdadm) from\nPlatform9 with the [etcd-manager](https://github.com/kopeio/etcd-manager)\nproject from kopeio / @justinsb.\n\netcdadm gives us easy to use CLI commands, which will form the base layer of\noperation.  Automation should ideally describe what it is doing in terms of\netcdadm commands, though we will also expose etcdadm as a go-library for easier\nconsumption, following the kubectl pattern of a `cmd/` layer calling into a\n`pkg/` layer.  This means the end-user can understand the operation of the\ntooling, and advanced users can feel confident that they can use the CLI tooling\nfor advanced operations.\n\netcd-manager provides automation of the common scenarios, particularly when\nrunning on a cloud.  It will be rebased to work in terms of etcdadm CLI\noperations (which will likely require some functionality to be added to etcdadm\nitself).  Where automation is not known to be safe, etcd-manager can stop and\nallow for manual intervention using the CLI.\n\nkops is currently using etcd-manager, and we aim to switch to the (new) etcadm asap.\n\nWe expect other tooling (e.g. cluster-api implementations) to adopt this project\nfor etcd management going forwards, and do a first integration or two if it\nhasn't happened already.\n\n### User Stories\n\n#### Manual Cluster Creation\n\nA cluster operator setting up a cluster manually will be able to do so using etcdadm and kubeadm.\n\nThe basic flow looks like:\n\n* On a master machine, run `etcdadm init`, making note of the `etcdadm join\n  \u003cendpoint\u003e` command\n* On each other master machine, copy the CA certificate and key from one of the\n  other masters, then run the `etcdadm join \u003cendpoint\u003e` command.\n* Run kubeadm following the [external etcd procedure](https://kubernetes.io/docs/setup/independent/high-availability/#external-etcd)\n\nThis results in an multi-node (\"HA\") etcd cluster.\n\n#### Automatic Cluster Creation\n\netcd-manager works by coordinating via a shared filesystem-like store (e.g. S3\nor GCS) and/or via cloud APIs (e.g. EC2 or GCE).  In doing so it is able to\nautomate the manual commands, which is very handy for running in a cloud\nenvironment like AWS or GCE.\n\nThe basic flow would look like:\n\n* The user writes a configuration file to GCS using `etcdadm seed\n  gs://mybucket/cluster1/etcd1 version=3.2.12 nodes=3`\n* On each master machine, run `etcdadm auto gs://mybucket/cluster1/etcd1`.\n  (Likely the user will have to run that persistently, either as a systemd\n  service or a static pod.)\n\n`etcdadm auto` downloads the target configuration from GCS, discovers other\npeers also running etcdadm, gossips with them to do basic leader election.  When\nsufficient nodes are available to form a quorum, it starts etcd.\n\n#### Automatic Cluster Creation with EBS volumes\n\netcdadm can also automatically mount EBS volumes.  The workflow looks like this:\n\n* As before, write a configuration file using `etcadm seed ...`, but this time\n  passing additional arguments \"--volume-tag cluster=mycluster\"\n* Create EBS volumes with the matching tags\n* On each master machine, run `etcdadm auto ...` as before.  Now etcdadm will\n  try to mount a volume with the correct tags before acting as a member of the\n  cluster.\n\n### Implementation Details/Notes/Constraints\n\n* There will be some changes needed to both platform9/etcdadm (e.g. etcd2\n  support) and kopeio/etcd-manager (to rebase on top of etcdadm).\n* It is unlikely that e.g. GKE / EKS will use etcdadm (at least initially),\n  which limits the pool of contributors.\n\n### Risks and Mitigations\n\n* Automatic mode may make incorrect decisions and break a cluster.  Mitigation:\n  automated backups, and a willingness to stop and wait for a fix / operator\n  intervention (CLI mode).\n* Automatic mode relies on peer-to-peer discovery and gossiping, which is less\n  reliable than Raft.  Mitigation: rely on Raft as much as possible, be very\n  conservative in automated operations (favor correctness over availability or\n  speed).  etcd non-voting members will make this much more reliable.\n\n## Graduation Criteria\n\netcdadm will be considered successful when it is used by the majority of OSS\ncluster installations.\n\n## Implementation History\n\n* Much SIG discussion\n* Initial proposal to SIG 2018-10-09\n* Initial KEP draft 2018-10-22 \n* Added clarification of cloud interaction 2018-10-23\n\n## Infrastructure Needed\n\n* etcdadm will be a subproject under sig-cluster-lifecycle\n"
+  },
+  {
+    "id": "e85e96e4fe8436a4262be99e7786d6f1",
+    "title": "Kubernetes Image Builder",
+    "authors": ["@timothysc", "@moshloop"],
+    "owningSig": "sig-cluster-lifecycle",
+    "participatingSigs": null,
+    "reviewers": ["@justinsb", "@luxas", "@astrieanna"],
+    "approvers": ["@justinsb", "@timothysc", "@luxas"],
+    "editor": "@timothysc",
+    "creationDate": "2019-06-11",
+    "lastUpdated": "2019-07-05",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Kubernetes Image Builder\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Phases](#phases)\n    - [Phase 0 (Base Image)](#phase-0-base-image)\n    - [Phase 1 (Software Installation / Customization)](#phase-1-software-installation--customization)\n    - [Phase 2 (Artifact Generation)](#phase-2-artifact-generation)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Infrastructure Needed](#infrastructure-needed)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Summary\nIt is common for modern cloud based software deployments to follow immutable patterns. One of the foundational pieces to this idea is the creation of immutable images. There are already several tools that create images in the Kubernetes ecosystem, which include: [Wardroom](https://github.com/heptiolabs/wardroom), [Cluster API AWS](https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/master/Makefile), [Cluster API vSphere](https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/master/Makefile), [amazon-eks-ami](https://github.com/awslabs/amazon-eks-ami), [talos](https://docs.talos-systems.com/), [LinuxKit](https://github.com/linuxkit/linuxkit),[kube-deploy](https://github.com/kubernetes/kube-deploy/tree/master/imagebuilder), etc. The purpose of this proposal is to distill down the common requirements and provide an image building utility that can be leveraged by the Kubernetes ecosystem.  \n\nThe purpose of this document is to request the creation of a sub-project of sig-cluster-lifecycle to address this space. \n\n## Motivation\nThere exists a need to be able to create repeatable IaaS images across providers for the explicit purpose of being able to deploy a Kubernetes cluster.\n\n### Goals \n* To build images for Kubernetes-conformant clusters in a consistent way across infrastructures, providers, and business needs.\n   * Install all software, containers, and configuration needed to pass conformance tests.\n   * Support end users requirements to customize images for their business needs.\n* To provide assurances in the binaries and configuration in images for purposes of security auditing and operational stability.\n   * Allow introspection of artifacts, software versions, and configurations in a given image.\n   * Support repeatable build processes where the same inputs of requested install versions result in the same installed binaries.\n* To ensure that the creation of images is performed via well defined phases.  Where users could choose specific phases that they needed.\n    * Support incremental usage.\n\n### Non-Goals \n* To publish images to cloud provider marketplaces, or to provide software workflow to automatically upload the built images on the cloud provider infrastructure.\n    * For example, it is not the responsibility of *this* utility to publish images to Amazon Marketplace. Each Cluster API Provider may implement its own image publisher. Users should be able to use the provider's publisher with the image output by the image builder.\n* To provide upgrade or downgrade semantics.\n* To provide guarantees that the software installed provides a fully functional system.\n* To prescribe the hardware architecture of the build system.\n* To create images from scratch.  \n    * The purpose of the tool is to take an existing image and make it Kubernetes ready.\n\n## Proposal \nThe Image Builder will start from one image in a supported format and create a new image in the same format specifically for the purpose of creating Kubernetes clusters.  In surveying the landscape of tooling it becomes readily apparent that there are a plethora of tools that provide an opinionated end-to-end user story around image creation, but we’ve observed it can be decomposed into a series of steps, or phases.  By decomposing the problem we can provide a rallying point for different tools to integrate, and provide the Kubernetes ecosystem with a common utility and user experience across those tools.\n\nAs a precondition the Image Builder will require a bootable disk image as an input, with native support for the cloud images published by the supported distributions.  However any external process or tool can be used to create the initial disk image from other sources including [ISO](https://packer.io)’s, file trees and [docker](https://github.com/iximiuz/docker-to-linux) images. Existing disk images can also be customized using tools like [virt-customize](http://libguestfs.org/virt-customize.1.html) before being fed into the Image Builder.\n\n**NOTE:** It should be noted that this document is intentionally high level and purposefully omits design choices which should be made at a later date once the subproject is further along in its lifecycle. \n\n### Phases \n#### Phase 0 (Base Image)\n\nLay down the initial base image.  Often times this can be some form of certified base image from a vendor or IT team.   **NOTE:** It is not a goal of this project to take on creation of those initial images.\n\n**Input:**\n\n`--disk-image` - A local or remote path to a libvirt/qemu supported image that a user or provider creates. (raw/qcow/vmdk etc.)\n\nImages are verified and cached by looking for a SHA256SUMS, sha256sum.txt file in the same directory as the image\n\n* [Ubuntu](https://cloud-images.ubuntu.com/bionic/current/)\n* [Fedora](https://alt.fedoraproject.org/cloud/)\n* [Debian](https://cloud.debian.org/images/openstack/) and a [comparison](https://wiki.debian.org/Cloud/SystemsComparison) of the types\n* [CentOS](https://cloud.centos.org/centos/7/images/)\n* [Amazon Linux](https://cdn.amazonlinux.com/os-images/current/kvm/)\n\n**Output:** Running shell inside the root filesystem\n\nPhase 0 will kickoff Phase 1, for example by chrooting into the disk or using nested virtualization to boot the image and then SSH into it.\n\n#### Phase 1 (Software Installation / Customization)\n\nThe purpose of this phase would be the installation of the Kubernetes stack, default account setup, updating packages, config, etc. \n\n**Input:** / with root/sudo access and a known package manager\n\n**Output:** A modified disk image\n\n#### Phase 2 (Artifact Generation)\n\nProduce output artifacts in their final form, and ideally this should include a BOM.\n\n### Risks and Mitigations\nGiven that there are already a plethora of existing solutions in the ecosystem the risk to the community is small, and this would allow contributors to help refine the best practices as they see them.  In the case where the subproject does not see traction we will orphan the subproject to the kubernetes-retired org. \n\n## Graduation Criteria \nalpha: Adoption across Cluster API providers.\n\n(post-alpha criteria will be added post-alpha)\n\n## Implementation History\nKEP created - Jun 12 2019 \nVote approved - Jul 02 2019 \n\n\n## Infrastructure Needed\nNone at this time, but it's possible this tool could become a critical piece of the test-automation for kubernetes, or Cluster API. \n\nWe are requesting to be a subproject under sig-cluster-lifecycle.\n\n## Alternatives \nPrior to this KEP a Cluster API workstream had written a [document](https://docs.google.com/document/d/1N65N1vCVa5QmU4BJXeSOImgRE8aq7daWhHt7XE9WCeI/edit?ts=5cde5f47#) outlining several options.\n"
+  },
+  {
+    "id": "9e2c0492ba0316ed4b23b8753cba12b4",
+    "title": "Kubernetes Bootstrap Checkpointing Proposal",
+    "authors": ["@timothysc"],
+    "owningSig": "sig-cluster-lifecycle",
+    "participatingSigs": ["sig-node"],
+    "reviewers": ["@yujuhong", "@luxas", "@roberthbailey"],
+    "approvers": ["@yujuhong", "@roberthbailey"],
+    "editor": "@timothysc",
+    "creationDate": "2017-10-20",
+    "lastUpdated": "2018-01-23",
+    "status": "implemented",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Kubernetes Bootstrap Checkpointing Proposal\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Objectives](#objectives)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Pod Submission to Running](#pod-submission-to-running)\n    - [Pod Deletion](#pod-deletion)\n    - [Cold Start](#cold-start)\n  - [Implementation Constraints](#implementation-constraints)\n- [Graduation Criteria](#graduation-criteria)\n- [Testing](#testing)\n- [Implementation History](#implementation-history)\n- [Unresolved Questions](#unresolved-questions)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThere are several methods to deploy a kubernetes cluster, one method that\noffers some unique advantages is self hosting.  The purpose of this proposal\nis to outline a method to checkpoint specific annotated pods, namely the\ncontrol plane components, for the purpose of enabling self hosting.\n\nThe details of self hosting are beyond the scope of this proposal, and are\noutlined in the references listed below:\n\n  - [Self Hosted Kubernetes][0]\n  - [Kubeadm Upgrades][1]\n\nExtra details on this proposal, and its history, can be found in the links\nbelow:\n\n  - [Bootstrap Checkpointing Draft 1][2]\n  - [Bootstrap Checkpointing Draft 2][3]\n  - [WIP Implementation][4]\n\n## Objectives\n\nThe scope of this proposal is **bounded**, but has the potential for broader\nreuse in the future.  The reader should be mindful of the explicitly stated\n[Non-Goals](#non-goals) that are listed below.\n\n### Goals\n\n - Provide a basic framework for recording annotated *Pods* to the filesystem.\n - Ensure that a restart of the kubelet checks for existence of these files\n and loads them on startup.\n\n### Non-Goals\n\n- This is not a generic checkpointing mechanism for arbitrary resources.\n(e.g. Secrets)  Such changes require wider discussions.\n- This will not checkpoint internal kubelet state.\n- This proposal does not cover self hosted kubelet(s).  It is beyond the\nscope of this proposal, and comes with it's own unique set of challenges.\n\n## Proposal\nThe enablement of this feature is gated by a single command line flag that\nis passed to the kubelet on startup, ```--bootstrap-checkpoint-path``` ,\nand will be denoted that it is ```[Alpha]```.\n\n### User Stories\n\n#### Pod Submission to Running\n- On submission of a Pod, via kubeadm or an operator, an annotation\n```node.kubernetes.io/bootstrap-checkpoint=true``` is added to that Pod, which\nindicates that it should be checkpointed by the kubelet.  When the kubelet\nreceives a notification from the apiserver that a new pod is to run, it will\ninspect the ```--bootstrap-checkpoint-path``` flag to determine if\ncheckpointing is enabled.  Finally, the kubelet will perform an atomic\nwrite of a ```Pod_UID.yaml``` file when the afore mentioned annotation exists.\nThe scope of this annotation is bounded and will not be promoted to a field.\n\n#### Pod Deletion\n- On detected deletion of a Pod, the kubelet will remove the associated\ncheckpoint from the filesystem.  Any failure to remove a pod, or file, will\nresult in an error notification in the kubelet logs.\n\n#### Cold Start\n- On a cold start, the kubelet will check the value of\n```--bootstrap-checkpoint-path```.  If the value is specified, it will read in\nthe contents of the that directory and startup the appropriate Pod.  Lastly,\nthe kubelet will then pull the list of pods from the api-server and rectify\nwhat is supposed to be running according to what is bound, and will go through\nits normal startup procedure.\n\n### Implementation Constraints\nDue to its opt-in behavior, administrators will need to take the same precautions\nnecessary in segregating master nodes, when enabling the bootstrap annotation.\n\nPlease see [WIP Implementation][4] for more details.\n\n## Graduation Criteria\n\nGraduating this feature is a responsibility of sig-cluster-lifecycle and\nsig-node to determine over the course of the 1.10 and 1.11 releases.  History\nhas taught us that initial implementations often have a tendency overlook use\ncases and require refinement.  It is the goal of this proposal to have an\ninitial alpha implementation of bootstrap checkpoining in the 1.9 cycle,\nand further refinement will occur after we have validated it across several\ndeployments.\n\n## Testing\nTesting of this feature will occur in three parts.\n- Unit testing of standard code behavior\n- Simple node-e2e test to ensure restart recovery\n- (TODO) E2E test w/kubeadm self hosted master restart recovery of an apiserver.\n\n## Implementation History\n\n- 20171020 - 1.9 draft proposal\n- 20171101 - 1.9 accepted proposal\n- 20171114 - 1.9 alpha implementation code complete\n\n## Unresolved Questions\n\n* None at this time.\n\n[0]: /contributors/design-proposals/cluster-lifecycle/self-hosted-kubernetes.md\n[1]: https://github.com/kubernetes/community/pull/825\n[2]: https://docs.google.com/document/d/1hhrCa_nv0Sg4O_zJYOnelE8a5ClieyewEsQM6c7-5-o/edit?ts=5988fba8#\n[3]: https://docs.google.com/document/d/1qmK0Iq4fqxnd8COBFZHpip27fT-qSPkOgy1x2QqjYaQ/edit?ts=599b797c#\n[4]: https://github.com/kubernetes/kubernetes/pull/50984\n"
+  },
+  {
+    "id": "ea1a92aade22d0c0a382907657d66464",
+    "title": "Kubeadm Config versioning",
+    "authors": ["@liztio"],
+    "owningSig": "sig-cluster-lifecycle",
+    "participatingSigs": null,
+    "reviewers": ["@timothysc"],
+    "approvers": ["TBD"],
+    "editor": "TBD",
+    "creationDate": "2018-04-12",
+    "lastUpdated": "2018-04-12",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Kubeadm Config Versioning\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories [optional]](#user-stories-optional)\n    - [As a user upgrading with Kubeadm, I want the upgrade process to not fail with unfamiliar configuration.](#as-a-user-upgrading-with-kubeadm-i-want-the-upgrade-process-to-not-fail-with-unfamiliar-configuration)\n    - [As a infrastructure system using kubeadm, I want to be able to write configuration files that always work.](#as-a-infrastructure-system-using-kubeadm-i-want-to-be-able-to-write-configuration-files-that-always-work)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nKubeadm uses MasterConfiguraton for two distinct but similar operations: Initialising a new cluster and upgrading an existing cluster. \nThe former is typically created by hand by an administrator. \nIt is stored on disk and passed to `kubeadm init` via command line flag.\nThe latter is produced by kubeadm using supplied configuration files, command line options, and internal defaults.\nIt will be stored in a ConfigMap so upgrade operations can find. \n\nRight now the configuration format is unversioned.\nThis means configuration file formats can change between kubeadm versions and there's no safe way to update the configuration format.\n\nWe propose a stable versioning of this configuration, `v1alpha2` and eventually `v1beta1`. \nVersion information will be _mandatory_ going forward, both for user-generated configuration files and machine-generated configuration maps.\n\nThere as an [existing document][config] describing current Kubernetes best practices around component configuration.\n\n[config]: https://docs.google.com/document/d/1FdaEJUEh091qf5B98HM6_8MS764iXrxxigNIdwHYW9c/edit#heading=h.nlhhig66a0v6\n\n## Motivation\n\nAfter 1.10.0, we discovered a bug in the upgrade process. \nThe `MasterConfiguraton` embedded a [struct that had changed][proxyconfig], which caused a backwards-incompatible change to the configuration format. \nThis caused `kubeadm upgrade` to fail, because a newer version of kubeadm was attempting to deserialise an older version of the struct.\n\nBecause the configuration is often written and read by different versions of kubeadm compiled by different versions of kubernetes, \nit's very important for this configuration file to be well-versioned. \n\n[proxyconfig]: https://github.com/kubernetes/kubernetes/commit/57071d85ee2c27332390f0983f42f43d89821961\n\n### Goals\n\n* kubeadm init fails if a configuration file isn't versioned\n* the config map written out contains a version\n* the configuration struct does not embed any other structs\n* existing configuration files are converted on upgrade to a known, stable version\n* structs should be sparsely populated\n* all structs should have reasonable defaults so an empty config is still sensible\n\n### Non-Goals\n\n* kubeadm is able to read and write configuration files for older and newer versions of kubernetes than it was compiled with\n* substantially changing the schema of the `MasterConfiguration`\n\n## Proposal\n\nThe concrete proposal is as follows.\n\n1. Immediately start writing Kind and Version information into the `MasterConfiguraton` struct.\n2. Define the previous (1.9) version of the struct as `v1alpha1`.\n3. Duplicate the KubeProxyConfig struct that caused the schema change, adding the old version to the `v1alpha1` struct.\n3. Create a new `v1alpha2` directory mirroring the existing [`v1alpha1`][v1alpha1], which matches the 1.10 schema. \n   This version need not duplicate the file as well.\n2. Warn users if their configuration files do not have a version and kind\n4. Use [apimachinery's conversion][conversion] library to design migrations from the old (v1alpha1) versions to the new (v1alpha2) versions\n5. Determine the changes for v1beta1\n6. With v1beta1, enforce presence of version numbers in config files and ConfigMaps, erroring if not present.\n\n[conversion]: https://godoc.org/k8s.io/apimachinery/pkg/conversion\n[v1alpha1]: https://github.com/kubernetes/kubernetes/tree/d7d4381961f4eb2a4b581160707feb55731e324e/cmd/kubeadm/app/apis/kubeadm \n\n### User Stories [optional]\n\n#### As a user upgrading with Kubeadm, I want the upgrade process to not fail with unfamiliar configuration.\n\nIn the past, the haphazard nature of the versioning system has meant it was hard to provide strong guarantees between versions.\nImplementing strong version guarantees mean any given configuration generated in the past by kubeadm will work with a future version of kubeadm. \nDeprecations can happen in the future in well-regulated ways.\n\n#### As a infrastructure system using kubeadm, I want to be able to write configuration files that always work.\n\nHaving a configuration file that changes without notice makes it very difficult to write software that integrates with kubeadm. \nBy providing strong version guarantees, we can guarantee that the files these tools produce will work with a given version of kubeadm.\n\n### Implementation Details/Notes/Constraints\n\nThe incident that caused the breakage in alpha wasn't a field changed it Kubeadm, it was a struct [referenced][struct] inside the `MasterConfiguration` struct.\nBy completely owning our own configuration, changes in the rest of the project can't unknowingly affect us.\nWhen we do need to interface with the rest of the project, we will do so explicitly in code and be protected by the compiler.\n\n[struct]: https://github.com/kubernetes/kubernetes/blob/d7d4381961f4eb2a4b581160707feb55731e324e/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go#L285\n\n### Risks and Mitigations\n\nMoving to a strongly versioned configuration from a weakly versioned one must be done carefully so as not break kubeadm for existing users. \nWe can start requiring versions of the existing `v1alpha1` format, issuing warnings to users when Version and Kind aren't present.\nThese fields can be used today, they're simply ignored.\nIn the future, we could require them, and transition to using `v1alpha1`.\n\n## Graduation Criteria\n\nThis KEP can be considered complete once all currently supported versions of Kubeadm write out `v1beta1`-version structs.\n\n## Implementation History\n\n## Alternatives\n\nRather than creating our own copies of all structs in the `MasterConfiguration` struct, we could instead continue embedding the structs.\nTo provide our guarantees, we would have to invest a lot more in automated testing for upgrades.\n"
+  },
+  {
+    "id": "fd886190b3298f06bcea7c0d1a92d820",
+    "title": "kubeadm join --control-plane workflow",
+    "authors": ["@fabriziopandini"],
+    "owningSig": "sig-cluster-lifecycle",
+    "participatingSigs": null,
+    "reviewers": ["@chuckha", "@detiber", "@luxas"],
+    "approvers": ["@luxas", "@timothysc"],
+    "editor": "@fabriziopandini",
+    "creationDate": "2018-01-28",
+    "lastUpdated": "2019-04-18",
+    "status": "provisional",
+    "seeAlso": ["KEP 0004"],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "# kubeadm join --control-plane workflow\n\n## Metadata\n\n```yaml\n```\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-goals](#non-goals)\n  - [Challenges and Open Questions](#challenges-and-open-questions)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Create a cluster with more than one control plane instance (static workflow)](#create-a-cluster-with-more-than-one-control-plane-instance-static-workflow)\n    - [Add a new control-plane instance (dynamic workflow)](#add-a-new-control-plane-instance-dynamic-workflow)\n  - [Implementation Details](#implementation-details)\n    - [Initialize the Kubernetes cluster](#initialize-the-kubernetes-cluster)\n    - [Dynamic workflow vs static workflow](#dynamic-workflow-vs-static-workflow)\n    - [Strategies for deploying control plane components](#strategies-for-deploying-control-plane-components)\n    - [Strategies for distributing cluster certificates](#strategies-for-distributing-cluster-certificates)\n    - [\u003ccode\u003ekubeadm upgrade\u003c/code\u003e for HA clusters](#-for-ha-clusters)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Drawbacks](#drawbacks)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Motivation\n\nSupport for high availability is one of the most requested features for kubeadm.\n\nEven if, as of today, there is already the possibility to create an HA cluster\nusing kubeadm in combination with some scripts and/or automation tools (e.g.\n[this](https://kubernetes.io/docs/setup/independent/high-availability/)), this KEP was\ndesigned with the objective to introduce an upstream simple and reliable solution for\nachieving the same goal.\n\nSuch solution will provide a consistent and repeatable base for implementing additional\ncapabilities like e.g. kubeadm upgrade for HA clusters.\n\n### Goals\n\n- \"Divide and conquer”\n\n  This proposal - at least in its initial release - does not address all the possible\n  user stories for creating an highly available Kubernetes cluster, but instead\n  focuses on:\n\n  - Defining a generic and extensible flow for bootstrapping a cluster with multiple control plane instances,\n    the `kubeadm join --control-plane` workflow.\n  - Providing a solution *only* for well defined user stories. see\n    [User Stories](#user-stories) and [Non-goals](#non-goals).\n\n- Enable higher-level tools integration\n\n  We expect higher-level and tooling will leverage on kubeadm for creating HA clusters;\n  accordingly, the `kubeadm join --control-plane` workflow should provide support for\n  the following operational practices used by higher level tools:\n\n  - Parallel node creation\n\n    Higher-level tools could create nodes in parallel (both nodes hosting control-plane instances and workers)\n    for reducing the overall cluster startup time.\n    `kubeadm join --control-plane` should support natively this practice without requiring\n    the implementation of any synchronization mechanics by higher-level tools.\n\n- Provide support both for dynamic and static bootstrap flow\n\n  At the time a user is running `kubeadm init`, they might not know what\n  the cluster setup will look like eventually. For instance, the user may start with\n  only one control plane instance + n nodes, and then add further control plane instances with\n  `kubeadm join --control-plane` or add more worker nodes with `kubeadm join` (in any order).\n  This kind of workflow, where the user doesn’t know in advance the final layout of the control plane\n  instances, into this document is referred as “dynamic bootstrap workflow”.\n\n  Nevertheless, kubeadm should support also more “static bootstrap flow”, where a user knows\n  in advance the target layout of the control plane instances (the number, the name and the IP\n  of nodes hosting control plane instances).\n\n- Support different etcd deployment scenarios, and more specifically run control plane components\n  and the etcd cluster on the same machines (stacked control plane nodes) or run the etcd\n  cluster on dedicated machines.\n\n### Non-goals\n\n- Installing a control-plane instance on an existing workers node.\n  The nodes must be created as a control plane instance or as workers and then are supposed to stick to the\n  assigned role for their entire life cycle.\n\n- This proposal doesn't include a solution for API server load balancing (Nothing in this proposal\n  should prevent users from choosing their preferred solution for API server load balancing).\n\n- This proposal doesn't include support for self-hosted clusters (but nothing in this proposal should\n  explicitly prevent us to reconsider this in the future as well).\n\n- This proposal doesn't provide an automated solution for transferring the CA key and other required\n  certs from one control-plane instance to the other. This is addressed in a separated KEP.\n  (see KEP [Certificates copy for join --control-plane](20190122-Certificates-copy-for-kubeadm-join--control-plane.md))\n\n- Nothing in this proposal should prevent practices that exist today.\n\n### Challenges and Open Questions\n\n- Keep the UX simple.\n\n  - _What are the acceptable trade-offs between the need to have a clean and simple\n    UX and the variety/complexity of possible kubernetes HA deployments?_\n\n- Create a cluster without knowing its final layout\n\n  Supporting a dynamic workflow implies that some information about the cluster are\n  not available at init time, like e.g. the number of control plane instances, the IP of\n  nodes candidates for hosting control-plane instances etc. etc.\n\n  - _How to configure a Kubernetes cluster in order to easily adapt to future change\n    of its own control plane layout like e.g. add a new control-plane instance, remove a\n    control plane instance?_\n\n  - _What are the \"pivotal\" cluster settings that must be defined before initializing\n    the cluster?_\n\n  - _How to combine into a single UX support for both static and dynamic bootstrap\n    workflows?_\n\n- Kubeadm limited scope of action\n\n  - Kubeadm binary can execute actions _only_ on the machine where it is running\n    e.g. it is not possible to execute actions on other nodes, to copy files across\n    nodes etc.\n  - During the join workflow, kubeadm can access the cluster _only_ using identities\n    with limited grants, namely `system:unauthenticated` or `system:node-bootstrapper`.\n\n- Upgradability\n\n  - How to setup an high available cluster in order to simplify the execution\n    of cluster version upgrades, both manually or with the support of `kubeadm upgrade`?_\n\n## Proposal\n\n### User Stories\n\n#### Create a cluster with more than one control plane instance (static workflow)\n\nAs a kubernetes administrator, I want to create a Kubernetes cluster with more than one\ncontrol-plane instances, of which I know in advance the name and the IP.\n\n\\* A new \"control plane instance\" is a new kubernetes node with\n`node-role.kubernetes.io/master=\"\"` label and\n`node-role.kubernetes.io/master:NoSchedule` taint; a new instance of control plane\ncomponents will be deployed on the new node; additionally, if the cluster uses local etcd mode,\nand etcd is created and managed by kubeadm, a new etcd member will be\ncreated on the joining machine as well.\n\n#### Add a new control-plane instance (dynamic workflow)\n\nAs a kubernetes administrator, (_at any time_) I want to add a new control-plane instance* to\nan existing Kubernetes cluster.\n\n### Implementation Details\n\n#### Initialize the Kubernetes cluster\n\nAs of today, a Kubernetes cluster should be initialized by running `kubeadm init` on a\nfirst node, afterward referred as the bootstrap control plane.\n\nin order to support the `kubeadm join --control-plane` workflow a new Kubernetes cluster is\nexpected to satisfy the following condition:\n\n- The cluster must have a stable `controlplaneAddress` endpoint (aka the IP/DNS of the\n  external load balancer)\n\nThe above condition/setting could be set by passing a configuration file to `kubeadm init`.\n\n#### Preparing for execution of kubeadm join --control-plane\n\nBefore invoking `kubeadm join --control-plane`, the user/higher level tools\nshould copy control plane certificates from an existing control plane instance, e.g. the bootstrap control plane\n\n\u003e NB. kubeadm is limited to execute actions *only*\n\u003e in the machine where it is running, so it is not possible to copy automatically\n\u003e certificates from remote locations.\n\u003e NB. https://github.com/kubernetes/enhancements/pull/713 is porposing a possible approach\n\u003e for automatic copy of certificates across nodes\n\nPlease note that strictly speaking only ca, front-proxy-ca certificate and service account key pair\nare required to be equal among all control plane instances. Accordingly:\n\n- `kubeadm join --control-plane` will check for the mandatory certificates and fail fast if\n  they are missing\n- given the required certificates exists, if some/all of the other certificates are provided\n  by the user as well, `kubeadm join --control-plane` will use them without further checks.\n- If any other certificates are missing, `kubeadm join --control-plane` will create them.\n\n\u003e see \"Strategies for distributing cluster certificates\" paragraph for\n\u003e additional info about this step.\n\n#### The kubeadm join --control-plane workflow\n\nThe `kubeadm join --control-plane` workflow will be implemented as an extension of the\nexisting `kubeadm join` flow.\n\n`kubeadm join --control-plane` will accept an additional parameter, that is the apiserver advertise\naddress of the joining node; as detailed in following paragraphs, the value assigned to\nthis parameter depends on the user choice between a dynamic bootstrap workflow or a static\nbootstrap workflow.\n\nThe updated join workflow will be the following:\n\n1. Discovery cluster info [No changes to this step]\n\n   \u003e NB This step waits for a first instance of the kube-apiserver to become ready\n   \u003e (the bootstrap control plane); And thus it acts as embedded mechanism for handling the sequence\n   \u003e `kubeadm init` and `kubeadm join` actions in case of parallel node creation.\n\n3. In case of `join --control-plane` [New step]\n\n   1. Using the bootstrap token as identity, read the `kubeadm-config` configMap\n      in `kube-system` namespace.\n\n      \u003e This requires to grant access to the above configMap for\n      \u003e `system:bootstrappers` group.\n\n   2. Check if the cluster/the node is ready for joining a new control plane instance:\n\n      a. Check if the cluster has a stable `controlplaneAddress`\n      a. Checks if the mandatory certificates exists on the file system\n\n   3. Prepare the node for hosting a control plane instance:\n\n      a. Create missing certificates (in any).\n         \u003e please note that by creating missing certificates kubeadm can adapt seamlessly\n         \u003e to a dynamic workflow or to a static workflow (and to apiserver advertise address\n         \u003e of the joining node). see following paragraphs for additional info.\n\n      a. Create static pod manifests for control-plane components and related kubeconfig files.\n\n      \u003e see \"Strategies for deploying control plane components\" paragraph\n      \u003e for additional info about this step.\n\n   4. Create the admin.conf kubeconfig file\n\n      \u003e This operation creates an additional root certificate that enables management of the cluster\n      \u003e from the joining node and allows a simple and clean UX for the final steps of this workflow\n      \u003e (similar to the what happen for `kubeadm init`).\n      \u003e However, it is important to notice that this certificate should be treated securely\n      \u003e for avoiding to compromise the cluster.\n\n3. Execute the kubelet TLS bootstrap process [No changes to this step]:\n\n4. In case of `join --control-plane` [New step]\n\n   1. In case of local etcd:\n\n      a. Create static pod manifests for etcd\n\n      b. Announce the new etcd member to the etcd cluster\n\n      \u003e Important! Those operations must be executed after kubelet is already started in order to minimize the time\n      \u003e between the new etcd member is announced and the start of the static pod running the new\n      \u003e etcd member, because during this time frame etcd gets temporary not available\n      \u003e (only when moving from 1 to 2 members in the etcd cluster).\n      \u003e From https://coreos.com/etcd/docs/latest/v2/runtime-configuration.html\n      \u003e \"If you add a new member to a 1-node cluster, the cluster cannot make progress before\n      \u003e the new member starts because it needs two members as majority to agree on the consensus.\n      \u003e You will only see this behavior between the time etcdctl member add informs the cluster\n      \u003e about the new member and the new member successfully establishing a connection to the existing one.\"\n\n      \u003e Important! In order to make possible adding a new etcd member, kubeadm is changing how local etcd is deployed\n      \u003e using the `--apiserver-advertise-address` as additional `listen-client-urls`;\n      \u003e Please note that re-using the `--apiserver-advertise-address` for etcd is a trade-off that\n      \u003e allows to keep the kubeadm UX simple, considering the possible alternative require the user to\n      \u003e specify another IP address for each joining control-plane node.\n\n      \u003e This decision introduce also a limitation, because HA with local etcd won't be supported when the\n      \u003e user sets the `--apiserver-advertise-address` of each kube-apiserver instance, including the\n      \u003e instance on the bootstrap control plane, _equal to the `controlplaneAddress` endpoint_.\n   \n   2. Apply master taint and label to the node.\n\n   3. Update the `kubeadm-config` configMap with the information about the new control plane instance.\n\n#### Dynamic workflow vs static workflow\n\nDefining how to manage the API server serving certificate is one of the most relevant decision\nthat impact how an Kubernetes cluster might change in future, because this certificate must include\nthe `--apiserver-advertise-address` for control plane instances.\n\nIn a static bootstrap workflow the final layout of the control plane - the number, the\nname and the IP of control plane nodes - is known in advance. As a consequence a possible approach\nis to add all the addresses of the control-plane nodes at `kubeadm init` time, and then\ndistribute the _same_ apiserver serving certificate among all the control plane instances.\n\nThis was the approach originally suggest in the kubeadm high availability guides, but this\nprevents to add _unknown_ control-plane instances to the cluster.\n\nInstead, the recommended approach suggested by this proposal is to let kubeadm take care of\nthe creation of _many_ API server serving certificates, one for on each node.\n\nAs described in the previous paragraph, if the apiserver serving certificate is missing,\n`kubeadm join --control-plane` will generate a new certificate on the joining\ncontrol-plane; however this certificate will be “almost equal” to the certificate created on\nthe bootstrap control plane, because it should consider the name/address of the joining node.\n\nAs a consequence, the new apiserver serving certificate is specific to the current node (it cannot\nbe reused on other nodes) but this is considered an acceptable trade-offs as it allows kubeadm to\nadapt seamlessly to a dynamic workflow or to a static workflow.\n\n#### Strategies for deploying control plane components\n\nAs of today kubeadm supports two solutions for deploying control plane components:\n\n1. Control plane deployed as static pods (current kubeadm default)\n2. Self-hosted control plane (currently alpha)\n\nAs stated above, supporting for HA self-hosted control plane is non goal for this\nproposal.\n\n#### Strategies for distributing cluster certificates\n\nAs of today kubeadm supports two solutions for storing cluster certificates:\n\n1. Cluster certificates stored on file system (current kubeadm default)\n2. Cluster certificates stored in secrets (currently alpha and applicable only to\n   self-hosted control plane)\n\nThere are two possible alternatives for case 1. \"Cluster certificates stored on file system\":\n\n- delegate to the user/the higher level tools the responsibility to copy certificates\n  from an existing node, e.g. the bootstrap control plane, to the joining node _before_\n  invoking `kubeadm join --control-plane`.\n\n- let kubeadm copy the certificates. This alternative is described in a separated KEP\n  (see KEP [Certificates copy for join --control-plane](20190122-Certificates-copy-for-kubeadm-join--control-plane.md))\n\nAs stated above, supporting for Cluster certificates self-hosted control plane is a non goal\nfor this proposal, and the same apply to Cluster certificates stored in secrets.\n\n#### `kubeadm upgrade` for HA clusters\n\nThe `kubeadm upgrade` workflow as of today is composed by two high level phases, upgrading the\ncontrol plane and upgrading nodes.\n\nThe above hig-level workflow will remain the same also in case of clusters with more than\none control plane instances, but with a new sub-step to be executed on secondary control-plane instances:\n\n1. Upgrade the control plane\n\n   1. Run `kubeadm upgrade apply` on a first control plane instance [No changes to this step]\n   1. Run `kubeadm upgrade node` on secondary control-plane instances [modified to make additional\n      steps in case of secondary control-plane nodes vs \"simple\" worker nodes]\n\n      Nb. while this feature will be alpha, there will be a separated `kubeadm upgrade node experimental-control-plane`\n      command.\n\n1. Upgrade nodes/kubelet [No changes to this step]\n\n## Graduation Criteria\n\n- To create a periodic E2E tests for HA clusters creation\n- To create a periodic E2E tests to ensure upgradability of HA clusters\n- To document the kubeadm support for HA in kubernetes.io\n\n## Implementation History\n\n- original HA proposals [#1](https://goo.gl/QNtj5T) and [#2](https://goo.gl/C8V8PV)\n- merged [Kubeadm HA design doc](https://goo.gl/QpD5h8)\n- HA prototype [demo](https://goo.gl/2WLUUc) and [notes](https://goo.gl/NmTahy)\n- [PR #58261](https://github.com/kubernetes/kubernetes/pull/58261) with the showcase implementation of the first release of this KEP\n- v1.12 first implementation of `kubeadm join --control plane`\n- v1.13 support for local/stacked etcd\n- v1.14 implementation of automatic certificates copy\n  (see KEP [Certificates copy for join --control-plane](20190122-Certificates-copy-for-kubeadm-join--control-plane.md)).\n\n## Drawbacks\n\nThe `kubeadm join --control-plane` workflow requires that some condition are satisfied at `kubeadm init` time,\nbut the user will be informed about compliance/non compliance of such conditions only when running \n`kubeadm join --control plane`.\n\n## Alternatives\n\n1) Execute `kubeadm init` on many nodes\n\nThe approach based on execution of `kubeadm init` on each node candidate for hosting a control plane instance\nwas considered as well, but not chosen because it seems to have several drawbacks:\n\n- There is no real control on parameters passed to `kubeadm init` executed on different nodes,\n  and this might lead to unpredictable inconsistent configurations.\n- The init sequence for above nodes won't go through the TLS bootstrap process,\n  and this might be perceived as a security concern.\n- The init sequence executes a lot of steps which are un-necessary (on an existing cluster); now those steps are\n  mostly idempotent, so basically now no harm is done by executing them two or three times. Nevertheless, to\n  maintain this contract in future could be complex.\n\nAdditionally, by having a separated `kubeadm join --control-plane` workflow instead of a single `kubeadm init`\nworkflow we can provide better support for:\n\n- Steps that should be done in a slightly different way on a secondary control plane instances with respect\n  to the bootstrap control plane (e.g. updating the kubeadm-config map adding info about the new control plane\n  instance instead of creating a new configMap from scratch).\n- Checking that the cluster/the kubeadm-config is properly configured for many control plane instances\n- Blocking users trying to create secondary control plane instances on clusters with configurations\n  we don't want to support as a SIG (e.g. HA with self-hosted control plane)\n"
+  },
+  {
+    "id": "63fd0c6654abe817122510266e9f44f1",
+    "title": "Kubeadm config file graduation (v1beta2)",
+    "authors": ["@fabriziopandini", "@luxas"],
+    "owningSig": "sig-cluster-lifecycle",
+    "participatingSigs": null,
+    "reviewers": ["@chuckha", "@detiber", "@liztio"],
+    "approvers": ["@luxas", "@timothysc", "@fabriziopandini", "@neolit123"],
+    "editor": "@fabriziopandini",
+    "creationDate": "2018-08-01",
+    "lastUpdated": "2019-04-29",
+    "status": "implementable",
+    "seeAlso": ["KEP 0008"],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# kubeadm Config file graduation (v1beta2)\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [v1beta1](#v1beta1)\n    - [Decoupling the kubeadm types from other ComponentConfig types](#decoupling-the-kubeadm-types-from-other-componentconfig-types)\n    - [Re-design how kubeadm configurations are persisted](#re-design-how-kubeadm-configurations-are-persisted)\n    - [Use substructures instead of the old \u0026quot;single flat object\u0026quot;](#use-substructures-instead-of-the-old-single-flat-object)\n  - [v1beta2](#v1beta2)\n    - [Add config options for new and existing kubeadm features](#add-config-options-for-new-and-existing-kubeadm-features)\n  - [v1beta3](#v1beta3)\n    - [Make kubeadm's config format more CRD and third party friendly](#make-kubeadms-config-format-more-crd-and-third-party-friendly)\n    - [Opt-in AddOns](#opt-in-addons)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n  - [v1alpha3 released with Kubernetes 1.12](#v1alpha3-released-with-kubernetes-112)\n  - [v1beta1 released with Kubernetes 1.13](#v1beta1-released-with-kubernetes-113)\n  - [v1beta2 released with Kubernetes 1.15](#v1beta2-released-with-kubernetes-115)\n- [Drawbacks](#drawbacks)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis KEP is meant to describe design goal and the proposed solution for implementing the kubeadm\nconfig file format.\n\nThe kubeadm config file today is one of the first touch points with Kubernetes for many users and\nalso for higher level tools leveraging kubeadm; as a consequence, providing a more stable and\nreliable config file format is considered one of the top priorities for graduating kubeadm itself\nto GA and for the future of kubeadm itself.\n\n## Motivation\n\nThe kubeadm config file is a set of YAML documents with versioned structs that follow the Kubernetes\nAPI conventions with regards to `apiVersion` and `kind`, but these types aren’t exposed as an\nAPI endpoint in the API server. kubeadm follows the ComponentConfig conventions.\n\nThe kubeadm config file was originally created as alternative to command line flags for `kubeadm init`\nand `kubeadm join` actions, but over time the number of options supported by the kubeadm config file\nhas grown continuously, while the number of command line flags is intentionally kept under control\nand limited to the most common and simplest use cases.\n\nAs a consequence today the kubeadm config file is the only viable way for implementing many use cases\nlike e.g. usage of an external etcd, customizing Kubernetes control plane components or kube-proxy\nand kubelet parameters.\n\nAdditionally, the kubeadm config file today acts also as a persistent representation of the cluster\nspecification that can be used at a any points in time after `kubeadm init` e.g. for executing\n`kubeadm upgrade` actions.\n\n### Goals\n\n- To provide a solution for decoupling the kubeadm ComponentConfig types from other Kubernetes\n  components’ ComponentConfig types.\n\n- To re-design how kubeadm configurations are persisted, addressing the known limitations/weakness\n  of the current design; more in detail, with the aim to provide a better support for high\n  availability clusters, it should be provided a clear separation between cluster wide settings,\n  control plane instance settings, node/kubelet settings and runtime settings (setting used\n  by the current command but not persisted).\n\n- Improve the current kubeadm config file format by using specialized substructures instead\n  of the old \"single flat object with only fields\".\n\n### Non-Goals\n\n- To steer/coordinate all the implementation efforts for adoption of ComponentConfig across all\n  the different Kubernetes components.\n\n- To define a new home for the Bootstrap Token Go structs.\n\n- To address all possible proposed changes, that were proposed to the kubeadm config format.\n  Specifically we are trying to avoid changes, that have a fairly limited use cases.\n\n## Proposal\n\n### v1beta1\n\nThis section outlines proposed changes to be made into v1beta1, possibly appearing in stages\nin alpha versions between v1alpha2 and v1beta1.\n\n#### Decoupling the kubeadm types from other ComponentConfig types\n\nThe `v1alpha2` kubeadm config types embeds the ComponentConfig for\nkube-proxy and kubelet into the **MasterConfiguration** object; it is expected that also\nComponentConfig for kube-controller manager, kube-scheduler and kube-apiserver will be added\nin future (non goal of this KEP).\n\nThis strong type of dependency - embeds - already created some problem in the v1.10 cycle, and\ndespite some improvements, the current situation is not yet ideal, because e.g embedded dependency\ncould impact the kubeadm config file life cycle, forcing kubeadm to change its own config file\nversion every time one of the embedded component configurations changes.\n\n`v1beta1` config file version is going to address this problem by removing embedded dependencies\nfrom the _external_ kubeadm config types.\n\nInstead, the user will be allowed to pass other component’s ComponentConfig in separated YAML\ndocuments inside of the same YAML file given to `kubeadm init --config`.\n\n\u003e please note that the _kubeadm internal config_ might continue to embed components config\n\u003e for some time into the future because kubeadm uses its internal config to propagate the knowledge\n\u003e of such data structures e.g. for propagating network configuration settings to kubelet, setting\n\u003e defaults, validating or manipulating YAML etc.\n\n#### Re-design how kubeadm configurations are persisted\n\nIn `v1alpha2` the kubeadm **MasterConfiguration** struct is persisted as a whole into the\n`kubeadm-config` ConfigMap, but this situation has well know limitations/weaknesses:\n\n- There is no clear distinction between cluster wide settings (e.g. the kube-apiserver server\n  extra-args that should be consistent across all instances) and control plane instance settings\n  (e.g. the API server advertise address of a kube-apiserver instance).\n  NB. This is currently the main blocker for implementing support for high availability clusters\n  in kubeadm.\n\n- There is no clear distinction between cluster wide settings and node/kubelet specific\n  settings (e.g. the node name of the current node)\n\n- There is no clear distinction between cluster wide settings and runtime configurations\n  (e.g. the token that should be created by kubeadm init)\n\n- ComponentConfigs are stored both in the `kubeadm-config` and in the `kubeproxy-config` and\n  `kubelet-config-vX.Y` ConfigMaps, with the first used as authoritative source for updates,\n  while the others are the one effectively used by components.\n\nConsidering all the above points, and also the split of the other components ComponentConfigs\nfrom the kubeadm **MasterConfiguration** type described in the previous paragraph, it should\nbe re-designed how kubeadm configuration is persisted.\n\nThe proposed solution leverage on the new kubeadm capability to handle separated YAML documents\ninside of the same kubeadm-config YAML file. More in detail:\n\n- **MasterConfiguration** will be split into two other top-level kinds: **InitConfiguration**\n  and **ClusterConfiguration**.\n- **InitConfiguration** will host the node-specific options like the node name, kubelet CLI flag\n  overrides locally, and ephemeral, init-only configuration like the Bootstrap Tokens to initialize\n  the cluster with.\n- **ClusterConfiguration** will host the cluster-wide configuration, and **ClusterConfiguration**\n  is the object that will be stored in the `kubeadm-config` ConfigMap.\n- Additionally, **NodeConfiguration** will be renamed to **JoinConfiguration** to be consistent with\n  **InitConfiguration** and highlight the coupling to the `kubeadm join` command and its\n  ephemeral nature.\n\nThe new `kubeadm init` flow configuration-wise is summarized by the attached schema.\n\n![kubeadm-init](0023-kubeadm-init.png)\n\n[link](0023-kubeadm-init.png)\n\nAs a consequence, also how the kubeadm configuration is consumed by kubeadm commands should\nbe adapted as described by following schemas:\n\n- [kubeadm join and kubeadm join --master](0023-kubeadm-join.png)\n- [kubeadm upgrade apply](0023-kubeadm-upgrade-apply.png)\n- [kubeadm upgrade node](0023-kubeadm-upgrade-node.png)\n- [kubeadm reset](0023-kubeadm-reset.png)\n\n#### Use substructures instead of the old \"single flat object\"\n\nEven if with few exceptions, the kubeadm **MasterConfiguration** and **NodeConfiguration** types\nin `v1alpha1` and `v1alpha2` are basically single, flat objects that holds all the configuration\nsettings, and this fact e.g. doesn’t allow to a clearly/easily understand which configuration\noptions relate to each other or apply to the different control plane components.\n\nWhile redesigning the config file for addressing the main issues described in previous paragraphs,\nkubeadm will provide also a cleaner representation of attributes belonging to single component/used\nfor a specific goal by creating dedicated objects, similarly to what’s already improved for\netcd configuration in the `v1alpha2` version.\n\n### v1beta2\n\nThis section outlines changes to be introduced in a second iteration of the kubeadm config format.\n\n#### Add config options for new and existing kubeadm features\n\nOver time kubeadm gains new features which may require the addition of new settings to the config\nformat. One notable such feature, that was introduced after the release of `v1beta1` is the\n[Certificates copy for join --control-plane KEP](./20190122-Certificates-copy-for-kubeadm-join--control-plane.md).\n\nIn addition to that, some new settings are added to the config format for existing features,\nimproving the original design and addressing user feedbacks.\nThese include an ability to specify the timeout that is used to wait for the API server to become\nactive (a setting already introduced in `v1beta1`) and an ability to ignore some pre-flight errors\n(a setting to be introduced in `v1beta2`).\n\nNevertheless, SIG Cluster Lifecycle should be careful not to add fields that are needed for fairly\nlimited use cases.\n\n### v1beta3\n\nThis section outlines changes to be introduced in the third iteration of the kubeadm config format.\n\n#### Make kubeadm's config format more CRD and third party friendly\n\nWith more and more external tools consuming and providing kubeadm configuration it is desirable to increase the compatibility with other tools.\nThis includes:\n- Adding metadata fields to InitConfiguration, JoinConfiguration and ClusterConfiguration.\n- Marking omitempty fields as `+optional`.\n\n#### Opt-in AddOns\n\nIn the past few releases users have been increasingly using the kubeadm phases feature to skip the installation of the Kube-Proxy and CoreDNS/Kube-DNS addons. This, however, causes some problems when joining new nodes to the cluster or upgrading existing ones, as there are no means of persisting the user wish to not install some of the addons.\nThis, combined with recent developments in the Cluster AddOns sub-project of SIG Cluster Lifecycle, led us to believe, that the best way to tackle the problem at hand is to allow for users to specify precisely which addons should be installed by kubeadm and persist the choice in the ClusterConfiguration.\n\n### Risks and Mitigations\n\nThis is a change mostly driven by kubeadm maintainers, without an explicit buy-in from customers\nusing kubeadm in large installations\n\nThe differences from the current config file are relevant and kubeadm users can get confused.\n\nAbove risks will be mitigated by:\n\n- providing a fully automated conversion mechanism and a set of utilities under the kubeadm\n  config command (a goal and requirement for this KEP)\n- The new structure could potentially make configuration options less discoverable as they’re\n  buried deeper in the code. Sufficient documentation for common and advanced tasks will help\n  mitigate this.\n- writing a blog post before the release cut\n- providing adequate instructions in the release notes\n\nImpact on the code is considerable.\n\nThis risk will be mitigated by implementing the change according to following approach:\n\n- introducing a new `v1alpha3` config file as a intermediate step before `v1beta1`\n- implementing all the new machinery e.g. for managing multi YAML documents in one file, early\n  in the cycle\n- ensuring full test coverage about conversion from `v1alpha2` to `v1alpha3`, early in the cycle\n- postponing the final rename from `v1alpha3` to `v1beta1` only when all the graduation criteria\n  are met, or if this is not the case, iterating the above steps in following release cycles\n\n## Graduation Criteria\n\n- There is an upgrade path from earlier versions.\n- The primary kinds that can be serialized/deserialized are `InitConfiguration`,\n  `JoinConfiguration` and `ClusterConfiguration`.\n- ComponentConfig structs for other Kubernetes components are supplied besides\n  `ClusterConfiguration` in different YAML documents.\n- SIG Cluster Life cycle is happy with the structure of the types.\n\n## Implementation History\n\n### v1alpha3 released with Kubernetes 1.12\n\n- The kubeadm own types were decoupled from the other component config types.\n- Clear distinction was introduced between cluster wide settings (stored on a config map in the\n  cluster), node only and runtime settings.\n\n### v1beta1 released with Kubernetes 1.13\n\n- Focus on providing more structure of the config format, in contrast with large and mostly flat\n  structures that deal with different kinds of settings.\n- Improved the config format with respect to HA.\n\n### v1beta2 released with Kubernetes 1.15\n\n- Added new fields for specifying the encryption key for certificates and for specifying which pre-flight errors to be ignored.\n- **omitempty** has a wider use, but is removed from the *taints* field of NodeRegistrationOptions.\n\n## Drawbacks\n\nThe differences from the current kubeadm config are relevant and kubeadm users can get confused.\n\nThe impacts on the current codebase are considerable it is required an high commitment from\nthe SIG. This comes with a real opportunity cost.\n"
+  },
+  {
+    "id": "17ef5401811b10ae33fe3bd036c6326c",
+    "title": "kubeadm phases to beta",
+    "authors": ["@fabriziopandini"],
+    "owningSig": "sig-cluster-lifecycle",
+    "participatingSigs": null,
+    "reviewers": ["@chuckha", "@detiber", "@liztio", "@neolit123"],
+    "approvers": ["@luxas", "@timothysc"],
+    "editor": "@fabriziopandini",
+    "creationDate": "2018-03-16",
+    "lastUpdated": "2019-01-22",
+    "status": "provisional",
+    "seeAlso": ["KEP 0008"],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# kubeadm phases to beta\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n  - [Implementation Details](#implementation-details)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Drawbacks](#drawbacks)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nWe are defining the road map for graduating `kubeadm alpha phase` commands to\nbeta, addressing concerns/lessons learned so far about the additional\neffort for maintenance of this feature.\n\n## Motivation\n\nThe `kubeadm phase` command was introduced in v1.8 cycle under the `kubeadm alpha phase`\ncommand with the goal of providing users with an interface for invoking individually\nany task/phase of the `kubeadm init` workflow.\n\nDuring this period, `kubeadm phase` proved to be a valuable and composable\nAPI/toolbox that can be used by any IT automation tool or by an advanced user for\ncreating custom clusters.\n\nHowever, the existing separation of `kubeadm init` and `kubeadm phase` in the code base\nrequired a continuous effort to keep the two things in sync, with a proliferation of flags,\nduplication of code or in some case inconsistencies between the init and phase implementation.\n\n### Goals\n\n- To define the approach for graduating to beta the `kubeadm phase` user\n  interface.\n- To significantly reduces the effort for maintaining `phases` up to date\n  with `kubeadm init`.\n- To support extension of the \"phases\" concept to other kubeadm workflows.\n- To enable re-use of phases across different workflows e.g. the cert phase\n  used by the `kubeadm init` and by the `kubeadm join` workflows.\n\n### Non-goals\n\n- This proposal doesn't include any changes of improvements to the actual `kubeadm init`\n  workflow.\n- This proposal doesn't include a plan for implementation of phases in workflows\n  different than the `kubeadm init` workflow; such plans will be defined by the sig \n  during release planning for each cycle, and then documented in this KEP.\n  - v1.14 implementation of phases in the `kubeadm join` workflow\n  - v1.15 implementation of phases in the `kubeadm upgrade node` workflow\n  - v1.17 implementation of phases in the `kubeadm upgrade apply` workflow\n\n## Proposal\n\n### User Stories\n\n- As a kubernetes administrator/IT automation tool, I want to run all the phases of\n  the `kubeadm init` workflow.\n- As a kubernetes administrator/IT automation tool, I want to run only one or some phases\n  of the `kubeadm init` workflow.\n- As a kubernetes administrator/IT automation tool, I want to run all the phases of\n  the `kubeadm init` workflow except some phases.\n- Same as above, but for the `kubeadm join` workflow\n- As a kubernetes administrator/IT automation tool, I used kubeadm init/join and\n  excluded some phases, I will want to run all the phases of kubeadm upgrade except\n  for the equivalent phases.\n\n### Implementation Details\n\nThe core of the new phase design consist into a simple, lightweight workflow manager to be used\nfor implementing composable kubeadm workflows.\n\nComposable kubeadm workflows are build by an ordered sequence of phases; each phase can have it's\nown, nested, ordered sequence of phases. For instance:\n\n```bash\n  preflight       Run master pre-flight checks\n  certs           Generates all PKI assets necessary to establish the control plane\n    /ca             Generates a self-signed kubernetes CA \n    /apiserver      Generates an API server serving certificate and key\n    ...\n  kubeconfig      Generates all kubeconfig files necessary to establish the control plane \n    /admin          Generates a kubeconfig file for the admin to use and for kubeadm itself\n    /kubelet        Generates a kubeconfig file for the kubelet to use.\n    ...\n  ...\n````\n\nThe above list of ordered phases should be made accessible from all the command supporting phases\nvia the command help, e.g. `kubeadm init --help` (and eventually in the future `kubeadm join --help` and `kubeadm upgrade apply --help` etc.)\n\nAdditionally we are going to improve consistency between the command outputs/logs with the name of phases\ndefined in the above list. This will be achieved by enforcing that the prefix of each output/log should match\nthe name of the corresponding phase, e.g. `[certs/ca] Generated ca certificate and key.` instead of the current\n`[certificates] Generated ca certificate and key.`.\n\nSingle phases will be made accessible to the users via a new `phase` sub command that will be nested in the\ncommand supporting phases, e.g. `kubeadm init phase` and `kubeadm join phase` (and eventually in the future `kubeadm upgrade apply phase` etc.). e.g.\n\n```bash\nkubeadm init phases certs [flags]\n\nkubeadm init phases certs ca [flags]\n```\n\nAdditionally we are going also to allow users to skip phases from the main workflow, via the `--skip-phases` flag. e.g.\n\n```bash\nkubeadm init --skip-phases addons/proxy\n```\n\nThe above UX will be supported by a new components, the `PhaseRunner` that will be responsible\nof running phases according to the given order; nested phases will be executed\nimmediately after their parent phase.\n\nThe `PhaseRunner` will be instantiated by kubeadm commands with the configuration of the specific list of ordered\nphases; the `PhaseRunner` in turn will dynamically generate all the `phase` sub commands for the phases.\n\nPhases invoked by the `PhaseRunner` should be designed in order to ensure reuse across different\nworkflows e.g. reuse of phase `certs` in both `kubeadm init` and `kubeadm join` workflows.\n\n## Graduation Criteria\n\n* To create a periodic E2E test that bootstraps a cluster using phases\n* To document the new user interface for phases in kubernetes.io\n\n## Implementation History\n\n* [#61631](https://github.com/kubernetes/kubernetes/pull/61631) First prototype implementation \n  (now outdated)\n* v1.13 implementation of phases in the `kubeadm init` workflow\n* v1.14 implementation of phases in the `kubeadm join` workflow\n* v1.15 implementation of phases in the `kubeadm upgrade node` workflow\n* v1.17 implementation of phases in the `kubeadm upgrade apply` workflow (planned)\n\n## Drawbacks\n\nIt would not be possible to expose to the users phases which are not part of kubeadm workflows\n(\"extra\" phases should be hosted on dedicated `kubeadm alpha` command and follow the normal\ngraduation process).\n\nThis is considered an acceptable trade-off in light of the benefits of the suggested\napproach.\n\n## Alternatives\n\nIt is possible to graduate phases by simply moving corresponding command to first level,\nbut this approach provide less opportunities for reducing the effort\nfor maintaining phases up to date with the changes of kubeadm.\n"
+  },
+  {
+    "id": "c6513fdd9e273887c8cebbd3fb9c32de",
+    "title": "Certificates copy for join --control-plane",
+    "authors": ["@fabriziopandini"],
+    "owningSig": "sig-cluster-lifecycle",
+    "participatingSigs": null,
+    "reviewers": ["@neolit123", "@detiber", "@mattmoyer", "@chuckha", "@liztio"],
+    "approvers": ["@timothysc", "@luxas"],
+    "editor": "TBD",
+    "creationDate": "2019-01-22",
+    "lastUpdated": "2019-04-18",
+    "status": "implementable",
+    "seeAlso": ["KEP-0015"],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Certificates copy for join --control-plane\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Constraints](#constraints)\n  - [Key element of the proposal](#key-element-of-the-proposal)\n  - [Implementation Details](#implementation-details)\n    - [The encryption key](#the-encryption-key)\n    - [The kubeadm-certs secret](#the-kubeadm-certs-secret)\n    - [The TTL-token](#the-ttl-token)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [\u003cstrong\u003eTest\u003c/strong\u003e Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [x] k/enhancements [issue 357](https://github.com/kubernetes/enhancements/issues/357) in release\n      milestone and linked to KEP\n- [x] KEP approvers have set the KEP status to `implementable`\n- [x] Design details are appropriately documented\n- [x] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [x] Graduation criteria is in place\n- [x] \"Implementation History\" section is up-to-date for milestone\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG\n      meetings, relevant PRs/issues, release notes\n\n## Summary\n\nAutomatic certificates copy makes easier to create HA clusters with the kubeadm tool using exactly\nthe same `kubeadm init` and `kubeadm join` commands the users are familiar with.\n\n## Motivation\n\nAs confirmed by the recent [kubeadm survey](https://drive.google.com/file/d/1eN9sGsdXWpurmplbEVn9UX5NiseQqlzO/view?usp=sharing),\nsupport for high availability cluster is one of the most requested features for kubeadm.\n\nA lot of effort was already done in kubeadm for achieving this goal, among them the redesign\nof the kubeadm config file and its graduation to beta and the implementation of the\n[`kubeadm join --control-plane workflow (KEP0015)`](https://github.com/kubernetes/enhancements/blob/master/keps/sig-cluster-lifecycle/kubeadm/0015-kubeadm-join-control-plane.md),\nbut the solution currently in place stills requires the manual copy of cluster certificates from\nthe bootstrap control-plane node to secondary control-plane nodes.\n\nThis KEP introduces automatic certificates copy, eliminating the manual operation described\nabove and completing the kubeadm solution for creating HA clusters.\n\n### Goals\n\n- Usability: use exactly the same `kubeadm init` and `kubeadm join` commands the users are\n  familiar with.\n- Security: provide a solution that enables the secure copy of cluster certificates between\n  control-plane nodes\n\n### Non Goals\n\n- Handle certificate copy in case of External CA.\n  If users decide to use an external CA with kubeadm without providing the CA keys, they will be required to create\n  signed certificates and all the kubeconfig files (including X.509 certificates) for the control plane nodes.\n  This is needed, because kubeadm cannot generate any certificates without the required CA keys.\n\n## Proposal\n\n### Constraints\n\nThe solution described in this proposal is deeply influenced by following constraints:\n\n1. Kubeadm can execute actions only on the machine where it is running e.g. it is not\n   possible to execute actions on other nodes.\n2. `kubeadm init` and `kubeadm join` are separated actions (executed on separated machines/at\n   different times).\n3. During the join workflow, kubeadm can access the cluster only using identities with\n   limited grants, namely `system:unauthenticated` or `system:node-bootstrapper`.\n\n### Key element of the proposal\n\nAt a high level, the proposed solution can be summarized by the following workflow:\n\n**On control plane node 1:**\n\n```bash\nkubeadm init --upload-certs\n```\n\nThe new `--upload-certs` flag will trigger a new kubeadm init phase executing following\nactions:\n\n1. An encryption key (32bytes for key SHA-256) will be generated; such key **will never be**\n   **stored on cluster**\n\n2. Cluster certificates will be encrypted using the above key (using AES-256/GCM as method)\n\n3. Encrypted certificates will be stored in a new Kubernetes secret named `kubeadm-certs`;\n   it is important to notice that:\n\n   - This secret is the technical solution that provides a temporary bridge between\n     `kubeadm init` and `kubeadm join` commands/between different nodes.\n\n   - Without the encryption key, this secret contains a harmless bag of bytes.\n\n4. A second bootstrap token will be generated, with a shorter duration than the\n   join token (the TTL-token)\n\n5. The `ownerReference` field in the `kubeadm-certs` secret will be linked to the\n   TTL token, thus ensuring automatic deletion of the secret when the TTL-token gets\n   deleted by the `TokenCleaner` controller\n\n6. RBAC rules will be created to ensure access to the above config map to users\n   in the `system:node-bootstrapper` group (the bootstrap tokens).\n\n**On control plane node 2:**\n\n```bash\nkubeadm kubeadm join --control-plane --certificate-key={key from step above}\n```\n\nThe new `--certificate-key` will trigger following actions:\n\n1. The `kubeadm-certs` config will be retrieved from the cluster\n2. Cluster certificates will be decrypted using the provided key, and\n   then stored on the disk\n\n### Implementation Details\n\n#### The encryption key\n\nThe default lifecycle of the encryption key will be the following:\n\n- a random encryption-key (32bytes for key SHA-256) will be created\n  by `kubeadm init --upload-certs`\n- the encryption-key will be printed in the output of the `kubeadm init --upload-certs`\n  command, but never stored in the cluster\n- after some TTL the `kubeadm-certs` encrypted with the above key will be automatically\n  deleted, thus limiting the risks related to a possible theft of encryption-key (see\n  risks and mitigations for more detail).\n\nThe following variant of the default lifecycle will be supported:\n\n1. The user will be allowed to pre-generate an encryption-key and pass it to\n   `kubeadm init --upload-certs` command using the kubeadm config file\n2. The user will be allowed to hide the encryption-key from the kubeadm output\n   using the `skip-token-print` or (a similar flag)\n3. The user will be allowed to pass the encryption-key to `kubeadm join --control-plane`\n   command using the kubeadm config file instead of the `--certificate-key` flag\n4. The user will be allowed to generate a new encryption-key after the first one expires by\n   invoking the `kubeadm init phases upload-certs` command (the command will\n  re-create/override the existing `kubeadm-certs` secret as well)\n\n#### The kubeadm-certs secret\n\nThe `kubeadm-certs` secret will be stored in the `kube-system` namespace; RBAC rules\nfor ensuring access to the users in the `system:node-bootstrapper` group\n(the bootstrap tokens) will be created.\n\nThe `kubeadm-certs` secret `ownerReference` attribute will be set to the TTL-token\nUID as described in the example below; See [Garbage Collection](https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/)\nfor more details.\n\n```yaml\napiVersion: v1\nkind: Secret\nmetadata:\n  name: kubeadm-certs\n  namespace: kube-system\n  ownerReferences:\n  - apiVersion: v1\n    controller: true\n    blockOwnerDeletion: true\n    kind: Secret\n    name: bootstrap-token-abcdef\n    uid: 84e913dc-191a-11e9-96ae-0242f550e41f\ntype: Opaque\ndata:\n  ...\n```\n\nThe `kubeadm-certs` secrets will contain the cluster certificates encrypted\nusing the encryption key (32bytes, SHA-256) with with AES-256/GCM as method, and then\nbase64 encoded as usual.\n\nPlease note that the upload certs process defined in this KEP will always\nupload all the necessary certificates (regardless of the cluster architecture e.g.\nexternal-CA or external etcd), because the assumption is that, when executed with\nthe `--upload-certs` flag, kubeadm is delegated by the user to copy all the certificates\nrequired for joining a new control plane node.\nIn other words, it won't be possible to use kubeadm `--upload-certs` for copying a subset\nof certificates only.\n\nFor example, in a cluster with local etcd the following certs/keys will be copied:\n- cluster CA cert and key (`/etc/kubernetes/pki/ca.crt` and `/etc/kubernetes/pki/ca.key`)\n- Front proxy CA cert and key (`/etc/kubernetes/pki/front-proxy-ca.crt` and `/etc/kubernetes/pki/front-proxy-ca.key`)\n- service account signing key (`/etc/kubernetes/pki/sa.key` and `/etc/kubernetes/pki/sa.pub`)\n- Etcd CA ert and key (`/etc/kubernetes/pki/etcd/ca.crt` and `/etc/kubernetes/pki/etcd/ca.key`)\n\nPlease note that client certificates are not part of the above list of certificates\nbecause `kubeadm join --control-plane` workoflow generates new client certificates taking\ncare of adding SANS specifically for the joining node.\n\n#### The TTL-token\n\nThe TTL-token is a regular bootstrap token, with a short duration, but without\nany assigned usage or groups (see [Authenticating with Bootstrap Tokens](https://kubernetes.io/docs/reference/access-authn-authz/bootstrap-tokens/#bootstrap-token-secret-format)\nfor more details).\n\nThis token is created with the only purpose of triggering automatic deletion of\nthe `kubeadm-certs` secret when deleted (the two objects will be linked via\n`ownerReference` attributes).\n\nThe `TokenCleaner` controller will ensure automatic deletion of bootstrap tokens\nand therefore also deletion of the TTL-token.\n\n### Risks and Mitigations\n\nIn case the cluster administrator uses the `kubeadm init --upload-certs` flag,\nthe `kubeadm-certs` secret will store sensitive cluster certificates. The following\nsecurity measures apply to the `kubeadm-certs` secret:\n\n- the `kubeadm-certs` secret is only valid for a short period of time (with a duration\n  shorter than bootstrap tokens)\n- if an attacker get access to the cluster when the `kubeadm-certs` secret still\n  exists, the cluster certs contained in the secret are protected by RBAC rules\n- if an attacker get access to the cluster when the `kubeadm-certs` secret still\n  exists with an identity that is authorized to read the `kubeadm-certs` secret,\n  the cluster certificates contained in the secret will be protected by the\n  encryption algorithm (without the encryption key, the secret contains a harmless\n  bag of bytes)\n\nThe following security measures apply to the encryption key:\n\n- the encryption key will never be stored on the cluster by kubeadm\n- the encryption key will never be transmitted on the wire by kubeadm\n- the encryption key is implicitly valid only for a short period of time (because\n  when the `kubeadm-certs` secret is automatically deleted, the encryption key\n  became useless). Please note that this does not apply if the content of the\n  kubeadm-certs was already obtained, the encryption-key could later decode it\n  (see residual risk below).\n- it will be possible to hide the encryption key from the `kubeadm init` output\n  by using `--skip-token-print` or similar flag (but this requires the user\n  passing an encryption key to kubeadm using the kubeadm config file), thus\n  avoiding the key improperly registered in logs\n- it will be possible to use a config file for passing the encryption key to\n  `kubeadm join` command, thus avoiding the key improperly registered in\n  shell history\n\nAnother possible risk derives from kubeadm allowing the user to fully customize\ncontrol plane components, including also the possibility to remove the TokenCleaner\nfrom the list of enabled controllers in the kube-scheduler configuration, and, as\na consequence, disable the mechanisms that manages temporarily valid tokens and\n`kubeadm-certs` secret.\n\nA possible mitigation action for this risk is to enforce the TokenCleaner in case\nwhen the user opts in to the upload-certs workflow defined in this document.\n\nDespite all the above measures, there is still a residual risk in case an attacker\nmanages to steal the encryption key from the cluster administrator and to get access\nto the cluster with adequate rights before the `kubeadm-certs` secret is\nautomatically deleted.\n\nHowever, we assume this residual risk could be accepted by the cluster administrator\nin favor of the simplified UX offered by the automatic certificates copy during\nthe `kubeadm init` / `kubeadm join --control-plane` operations.\n\nNevertheless, as an additional mitigation action, the potential risk will be clearly\nstated both in the documentation and in the kubeadm output.\n\n## Design Details\n\n### **Test** Plan\n\nThis feature is going to be tested:\n\n- with unit tests\n- with e2e tests, if kubeadm E2E tests are implemented during the v1.14 cycle\n- with integration tests (using Kind, recently extended for supporting HA/multi node)\n\n### Graduation Criteria\n\n- gather feedback from users\n- examples of real world usages\n- properly document the new function and the residual risk\n- have CI/CD tests validating the `kubeadm init` / `kubeadm join --control-plane`\n  workflow with automatic certificates copy\n\n### Upgrade / Downgrade Strategy\n\nNot applicable (this KEP simplify the cluster bootstrap process, but does not affect\nhow upgrades/downgrades are managed)\n\n### Version Skew Strategy\n\nNot applicable (this KEP simplify the cluster bootstrap process, but does not\naffect component version or version skew policy)\n\n## Implementation History\n\n- 22 Jan 2019 - first release of this KEP\n- v1.14 implementation as alpha feature without\n  - Extension of the kubeadm config file for allowing usage of pre-generated certificate keys\n  - TokenCleaner enforcement\n  - E2E tests\n\n## Alternatives\n\nSeveral alternatives were considered:\n\n- a simpler version of this proposal was considered (without TTL), but discarded\n  in favor of a more secure approach\n- alternative solutions - without the need of the `kubeadm-certs` secret - were\n  considered as well:\n  - Client/server architecture, with a service component on a bootstrap control-plane;\n    this was discarded due to the higher complexity\n  - Creation of kubernetes job with the responsibility of reading certs from\n    a bootstrap control-plane; this was discarded due to to the limited authorization\n    of the identity used by the kubeadm join workflow\n"
+  },
+  {
+    "id": "9465c7ef2053ae358e09c5224f815a66",
+    "title": "Artifact Generation",
+    "authors": ["@klaven", "@ncdc"],
+    "owningSig": "sig-cluster-lifecycle",
+    "participatingSigs": ["sig-cluster-lifecycle", "sig-release"],
+    "reviewers": ["@ncdc", "@hoegaarden", "“@neolit123”", "“@ixdy”"],
+    "approvers": ["@timothysc"],
+    "editor": "@Klaven",
+    "creationDate": "2019-02-15",
+    "lastUpdated": "2019-02-27",
+    "status": "implementable",
+    "seeAlso": ["https://github.com/kubernetes/enhancements/pull/843"],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# package-generation\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Story 1](#story-1)\n    - [Story 2](#story-2)\n    - [Story 3](#story-3)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n    - [Examples](#examples)\n      - [Removing a deprecated flag](#removing-a-deprecated-flag)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Drawbacks](#drawbacks)\n- [Alternatives](#alternatives)\n- [Infrastructure Needed](#infrastructure-needed)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nImplementation of a standard, community supported and maintained process of generating and building rpm and deb packages.\n\n## Motivation\n\nToday, the tooling used to build kubernetes packages is separate from kubernetes itself. This can cause downstream divergence, and cannot be easily maintained by the kubernetes community as a whole. \n\nThe primary goal of this proposal is to simplify the lifecycle management of packaging by co-locating the sources and tooling for generating packages into the kubernetes/kubernetes repository. \n\n### Goals\n\nTo create a maintainable and configurable method of building kubernetes artifacts supported and maintained by the community.\n\n- Move packaging tools out of kubernetes/release and into kubernetes/kubernetes repository\n- Enable customization for downstream packaging \n    - Including the ability to bring their own source code/binaries and changelogs \n    - Stop hard-coding the binary download URL to dl.k8s.io\n- Enable community governance over packaging of the project.\n    - Will be governed by sig-release\n- Tightly couple the versioning of packing and the kubernetes/kubernetes release.\n- Ensure that the same packages that are run in CI are the ones that are released.\n- Dedupe the generation of packages (THERE CAN BE ONLY ONE!) \n- Have one config file that is the source of truth for packaging details for both .rpm and .deb packages.\n- Artifact generation is independent of source code compilation. \n\n### Non-Goals\n\n- A change in the release process. See this [KEP](https://github.com/kubernetes/enhancements/pull/843).\n    - For example, package signing and distribution are not covered in this document and are covered in the above KEP.\n\n- To fundamentally change the packages and their dependencies. All packages should remain consistent with what we build today. \n\n## Proposal\n\nAs outlined in the following issue: https://github.com/kubernetes/kubernetes/issues/71677\nMove all packaging into the kubernetes/kubernetes repo and create the following 3 building blocks to support it.\n- Create a configuration yaml file to enable the configuring of the built packages.\n- Create a Go lang app to parse the yaml config and submit simple commands to the container.\n- Build inside a docker container based off [`fpm`](https://github.com/jordansissel/fpm).\n\n### User Stories\n\n#### Story 1\n\nAs a developer I would like to be able to install the entire set of alpha or beta (.deb/.rpm) packages to stand up a development cluster.  These packages should be equivalent to the future release artifacts.  \n\n#### Story 2\n\nAs a third party vendor, I would like to be able to package kubernetes deliverable artifacts using the same tooling that kubernetes’ uses.\n\n#### Story 3\n\nAs a release manager I would like to enable simple and easy package generation that the community can manage. \n\n### Implementation Details/Notes/Constraints [optional]\n\n- Use an [`fpm`](https://github.com/jordansissel/fpm) based container to build packages\n    - [`nfpm`](https://github.com/goreleaser/nfpm) was originally investigated but lacked features\n- Use yaml to configure\n- Create a go binary whose only job is to consume the yaml, build the right file structure and issue commands to the image to build the artifacts. Effectively the glue.\n\n### Risks and Mitigations\n\nRisk: Confusion as to what to use when. Mitigation: Coordination across groups to ensure that the changes made in k/k are consumed by the release team. \n\n## Design Details\n\n### Test Plan\n\nIdeally these new packages will get tested inside the test pipeline. When [`KIND`](https://sigs.k8s.io/kind) goes to run kubeadm/kubelet/kubectl/etc tests, it should install the packages generated by these tools as they will be contained within the kubernetes/kubernetes repository.\n\nThis will give us a clear path to the package consumed when testing, enabling a more confident release.\n\n### Graduation Criteria\n\nN/A\n\n#### Examples\n\n1:\nAn independent developer wanting to generate packages in their own fork would be able to edit the config yaml in their fork, override a dependency name and then issue a command to the go packaging binary to generate the packages.\n`[go packing binary] --config=myconfig.yaml`\n\n2:\nBazel will be able to update the configuration yaml to dynamically update the configuration before simply executing commands directly against the go packing binary.\n\n##### Removing a deprecated flag\n\nN/A \n\n[conformance tests]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/conformance-tests.md\n\n### Upgrade / Downgrade Strategy\n\nN/A\n\n### Version Skew Strategy\n\nN/A\n\n## Implementation History\n\nN/A\n\n## Drawbacks\n\nN/A\n\n## Alternatives\n\nN/A\n\n## Infrastructure Needed\n\nN/A\n\n\n"
+  },
+  {
+    "id": "4e37f7584ca679b1ca47331c17d27e9a",
+    "title": "kubeadm-for-windows",
+    "authors": ["@ksubrmnn", "@neolit123", "@patricklang"],
+    "owningSig": "sig-windows",
+    "participatingSigs": ["sig-windows", "sig-cluster-lifecycle"],
+    "reviewers": ["@timothysc", "@michmike", "@fabriziopandini", "@rosti"],
+    "approvers": ["@timothysc"],
+    "editor": "@ksubrmnn",
+    "creationDate": "2019-04-24",
+    "lastUpdated": "2019-04-24",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Kubeadm for Windows\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Story 1](#story-1)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n    - [Kubeadm manages the kubelet start / stop as a service](#kubeadm-manages-the-kubelet-start--stop-as-a-service)\n    - [Kubeadm makes assumptions about systemd and Linux](#kubeadm-makes-assumptions-about-systemd-and-linux)\n    - [Windows vs Linux host paths](#windows-vs-linux-host-paths)\n    - [Kube-proxy deployment](#kube-proxy-deployment)\n    - [CNI plugin deployment](#cni-plugin-deployment)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Drawbacks](#drawbacks)\n  - [Non-identical flows for Linux and Windows](#non-identical-flows-for-linux-and-windows)\n- [Infrastructure Needed](#infrastructure-needed)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n**ACTION REQUIRED:** In order to merge code into a release, there must be an issue in [kubernetes/enhancements] referencing this KEP and targeting a release milestone **before [Enhancement Freeze](https://github.com/kubernetes/sig-release/tree/master/releases)\nof the targeted release**.\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e., [kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be updated for the enhancement to be released.\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nIn Kubernetes 1.14, official support was added for Windows Containers. However, there is no official tool available to users for joining a Windows node to a cluster. The current solution is to create a set of scripts to download Kubernetes binaries and write their config files, but this has proven to be cumbersome and is a huge pain point for Windows adoption. On Linux, Kubeadm is available to quickly join nodes to a cluster, and the intent of this KEP is to propose a design to implement the same functionality for Windows.\n\nIt should be noted that at this time, this document only proposes enablement of support for Windows worker nodes using kubeadm.\n\n## Motivation\n\nThe motivation for this KEP is to provide a tool to allow users to take a Windows machine and join it to an existing Kubernetes cluster with a single command. The user should also be able to reset the node with a single command. We will also create a tool to install prerequisites and enable kubeadm to bring the Windows node to a Ready state.\n\n### Goals\n\n* Create and maintain a Powershell script to install and run Kubernetes prerequisites on Windows\n* Support kubeadm join for Windows\n* Support kubeadm reset for Windows\n* Create and maintain a Powershell script to deploy Windows CNIs\n\n### Non-Goals\n\n* Installing the Container Runtime (e.g. Docker or containerd)\n* Implement kubeadm init for Windows (at this time)\n* Implement kubeadm join --control-plane for Windows (at this time)\n* Supporting upgrades using kubeadm upgrade for Windows (to be revisited for Beta)\n* Running kube-proxy as a DaemonSet on Windows (to be revisited for Beta)\n* Running Flannel as a DaemonSet on Windows (to be revisited for Beta)\n\n## Proposal\n\n### User Stories\n\n#### Story 1\n\nA user will have a Windows machine that they want to join to an existing Kubernetes cluster.\n\n1. The user will download a set of required binaries (kubelet, kube-proxy, kubeadm, kubectl, Flannel) using a script. The same script will also wrap kubeadm execution.\n\n2. The script will register kubelet and kube-proxy as Windows services.\n\n3. The script will upload a default ConfigMap to the cluster for the default Windows KubeletConfiguration if one does not already exist.\n\n4. The script will run \"kubeadm join ...\" to join the node to the cluster. In this step kubeadm will run preflight checks and proceed with the regular join process.\n\n5. kubeadm will restart the kubelet service using flags that kubeadm fed to the Windows service. kubeadm will proceed to bootstrap the node. After this process is finished the node should show with the status of NotReady.\n\n6. The same script will then configure FlannelD. The script will (re-)register FlannelD as a service and start it with the correct configuration, optionally using parameters from kubeadm.\n\n7. kube-proxy will do the same steps as FlannelD shortly after.\n\n8. The node status should be Ready.\n\n### Implementation Details/Notes/Constraints\n\n#### Kubeadm manages the kubelet start / stop as a service\n\nKubeadm has a “kubelet” phase that can start/stop the [kubelet service](https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/phases/kubelet/kubelet.go).\n\nKubeadm already supports running the kubelet as a Windows service. This support was added by these two PRs: [60144](https://github.com/kubernetes/kubernetes/pull/60144), [53553](https://github.com/kubernetes/kubernetes/pull/53553).\n\nThis existing support will be leveraged by this proposal.\n\nCurrently kubeadm join downloads a KubeletConfiguration \"component config\" from a ConfigMap (that is created during kubeadm init) and writes it on all joining nodes. Then each node proceeds to add it's overrides via kubelet flags (that take precedence over config). This process has to be modified so that this KubeletConfiguration is defaulted properly for Windows nodes. One proposed solution is to upload the defaulted KubeletConfiguration for Windows in a separate ConfigMap, but this solution is a subject to change.\n\n#### Kubeadm makes assumptions about systemd and Linux\n\nKubeadm is quite coupled to systemd and Linux at the moment.\nThis file shows some of the kubelet-flags that are Linux bound:\n[kubelet flags](https://github.com/kubernetes/kubernetes/blob/b8b689aae0d6879de5192b203590330a11c7b9e3/cmd/kubeadm/app/phases/kubelet/flags.go)\n\nThe following changes are required::\n* Implement proper cri-socket detection in kubeadm for Windows.\n* Omit flags such as “cgroup-driver”\n* Specify the correct “pod-infra-container-image”\n* Also omit writing systemd drop-in files on Windows in general.\n* Populate the Windows service dynamically.\n\nTo support Windows specific flags for the kubelet, there is a requirement to split kubeadm’s app/phases/kubelet/flags.go files into two:\n* app/phases/kubelet/flags_windows.go\n* app/phases/kubelet/flags_linux.go\n\nIn the case of Windows, we will be updating the registration of the kubelet as a Windows service, by changing its flags based on the flags kubeadm wants to use, complying with Windows specific defaults in kubeadm.\n\nThe existing process of passing the following files to the kubelet, will be leveraged:\n* /etc/kubernetes/bootstrap-kubelet.conf\n* /etc/kubernetes/kubelet.conf\n* /var/lib/kubelet/config.yaml\n\nWindows related adjustments to default paths might be required. \n\n#### Windows vs Linux host paths\n\nKubeadm makes a number of non-portable assumptions about paths. E.g. “/etc/kubernetes” is a hardcoded path in kubeadm.\n\nWe will use \"C:\\kubernetes\" to hold the paths that are normally created for Linux.\n\nWe need to evaluate the kubeadm codebase for such instances of non-portable paths - CRI sockets, Cert paths, etc. Such paths need to be defaulted properly in the kubeadm configuration API.\n\nA consolidated list of paths where kubeadm installs files to a set of paths needs to be created and updated to comply with the Windows OS model. At least a single PR against kubeadm will be required to modify the Windows defaults. \n\nLast, as new paths are created, restrictive Access Control Lists (ACL) for Windows should be applied. Golang does not convert Posix permissions to an appropriate Windows ACL, so an additional step is needed. See [mkdirWithACL from moby/moby](https://github.com/moby/moby/blob/e4cc3adf81cc0810a416e2b8ce8eb4971e17a3a3/pkg/system/filesys_windows.go#L103)) for an example. This step will be performed by kubeadm.\n\n#### Kube-proxy deployment\n\nOn Linux, kube-proxy is deployed as a DaemonSet in the kubeadm init phase. However, kube-proxy cannot run as a container in Windows since Windows does not support privileged containers. Kube-proxy should therefore be run as a Windows service so that it is restarted by windows control manager automatically and has lifecycle control.\n\nWe need to modify the Linux kube-proxy DaemonSet to not deploy on Windows nodes. A PR is already in flight for that [76327](https://github.com/kubernetes/kubernetes/pull/76327). *Merging this PR is mandatory for this proposal*.\n\nRunning kube-proxy as a Windows service from kubeadm is out of scope for this proposal. This is due to the fact that we don’t want the changes in kubeadm to be intrusive to the existing method of running kube-proxy as a DaemonSet on Linux. This can end up requiring an abstraction layer that is far from ideal.\n\nThe proposed Windows wrapper script that executes kubeadm will also manage the restart of the kube-proxy Windows service.\n\nLong term and ideally, kube-proxy should be run as a DaemonSet on Windows.\n\n#### CNI plugin deployment\n\nOn Linux, CNI plugins are deployed via kubectl and run as a DaemonSet. However, on Windows, CNI plugins need to run on the node, and cannot run in containers (again because Windows does not currently support privileged containers). Azure-CNI, win-bridge (compatible with kubenet), and Flannel all need the binary and config stored on the node.\n\nThis proposal plans for FlannelD as the default option. Currently, FlannelD has to be started before the kube-proxy Windows service is started. FlannelD creates an HNS network on the Windows host, and kube-proxy will crash if it cannot find the network. This should be fixed in the scope of this project so that kube-proxy will wait until the network comes up. Therefore, kube proxy can be started at any time. \n\nHowever, if FlannelD is deployed in VXLAN (Overlay) mode, then we need to rewrite the KubeProxyConfiguration with the correct Overlay specific values, and kube-proxy will need to read this config again. This is not true for Host-Gateway (L2Bridge) mode. The script will have a flag that allows users to choose between the two networking modes.\n\nIf the users wish to use a different plugin they will technically opt-out of the supported setup for kubeadm based Windows worker nodes in the Alpha release.\n\nLong term, any CNI plugin should be supported for kubeadm based Windows worker nodes.\n\n### Risks and Mitigations\n\n**Risk**: Versioning of the wrapper script can become complicated\n\nVersioning of the script per-Kubernetes version can become a problem if a certain new version diverges in terms of download URLs, flags and configuration.\n\n*Mitigation*: Use git branches to version the script in the repository where it is hosted.\n\n**Risk**: The wrapper script is planned to act as both a downloader and runner of the downloader binaries, which might cause scope and maintenance issues.\n\n*Mitigation*: Use separate scripts, the first one downloads the binaries and the wrapper/runner script then setups the environment. The user then executes the wrapper script.\n\nThe initial plan is to give the single script method a shot with different arguments that will execute the different stages (downloading, setting up the environment, deploying the CNI).\n\n**Risk**: Flannel or kube-proxy require special configuration that kubeadm does not handle.\n\n*Mitigation*: Allow the user to pass custom configuration files that the wrapper script can feed into the components in question.\n\n**Risk**: Failing or missing preflight checks on Windows\n\n*Mitigation*: The existing kubeadm codebase already has good abstraction in this regard. Still, a PR that makes some non-intrusive adjustments in _windows.go files might be required.\n\n**Risk**: Permissions on Windows paths that kubeadm generates can pose a security hole.\n\nkubeadm creates directories using MakeAll() and such directories are strictly Linux originated for the time being - such as /etc.\n\nOn Windows, the creation of such a path can result in sensitive files to be exposed without the right permissions.\n\n*Mitigation*: Provide a SecureMakeAll() func in kubeadm, that ensures secure enough permissions on both Windows \u0026 Linux, and replace usage of MakeAll()\n\n## Design Details\n\n### Test Plan\n\nE2e testing for kubeadm on Windows is still being planned.\nOne available option is to run “kubeadm join” periodically on Azure nodes and federate the reports to test-infra/testgrid.\nThe CI signal will be owned by SIG Windows.\n\n### Graduation Criteria\n\nThis proposal targets *Alpha* support for kubeadm based Windows worker nodes in the release of Kubernetes 1.16.\n\n\n##### Alpha -\u003e Beta Graduation\n\nKube-proxy and CNI plugins are run as Kubernetes pods.\nThe feature is maintained by active contributors.\nThe feature is tested by the community and feedback is adapted with changes.\nKubeadm join performs complete preflight checks on the host node\nE2e tests might not be complete but provide good signal.\nDocumentation is in a good state. Kubeadm documentation is edited to point to documentation provided by SIG Windows.\nKubeadm upgrade is implemented.\n\n##### Beta -\u003e GA Graduation\n\nThe feature is well tested and adapted by the community.\nE2e test provide sufficient coverage.\nDocumentation is complete.\n\n### Upgrade / Downgrade Strategy\n\nUpgrades and downgrades are out of scope for this proposal for 1.16 but will be revisited in future iterations.\n\n### Version Skew Strategy\n\nThe existing version skew strategy will apply to Windows worker nodes using kubeadm.\nThe download scripts will not allow or recommend skewing the version of kube-proxy or the kubelet from the version of kubeadm that is installed by the user.\nIf the users applies manual skew by diverging from the recommended setup, the node will be claimed as unsupported.\n\n## Implementation History\n\n* April 2019 (1.14) KEP was created. \n* May 1, 2019       KEP was updated to address PR feedback\n* May 3, 2019       KEP was updated to address PR feedback\n* May 17, 2019      [PR 77989](https://github.com/kubernetes/kubernetes/pull/77989) Remove Powershell dependency for IsPrivilegedUser check on Windows\n* May 24, 2019      [PR 78053](https://github.com/kubernetes/kubernetes/pull/78053) Implement CRI detection for Windows\n* May 29, 2019      [PR 1136](https://github.com/coreos/flannel/pull/1136) Add net-config-path to FlannelD\n* May 31, 2019      [PR 78189](https://github.com/kubernetes/kubernetes/pull/78189) Use Service Control Manager as the Windows Initsystem\n* June 3, 2019      [PR 78612](https://github.com/kubernetes/kubernetes/pull/78612) Remove dependency on Kube-Proxy to start after FlannelD\n* July 20,2019      KEP was updated to target Alpha for 1.16\n\n\n## Drawbacks \n\n### Non-identical flows for Linux and Windows\nThere is overhead to maintaining two different methods for joining a node to a cluster depending on the operating system. However, this is necessary to meet the needs of a typical Windows customer.\n\n## Infrastructure Needed\n\nSIG Windows to provide:\n* Azure based infrastructure for testing kubeadm worker nodes.\n\nSIG Windows has provided:\n* A kubernetes/ org based repository to host the download / wrapper script: [sig-windows-tools](https://github.com/kubernetes-sigs/sig-windows-tools)\n"
+  },
+  {
+    "id": "3d63819c11457ac53e91abbf3b65fc32",
+    "title": "kubeadm-machine-output",
+    "authors": ["@akutz", "@bart0sh"],
+    "owningSig": "sig-cluster-lifecycle",
+    "participatingSigs": null,
+    "reviewers": [
+      "@justinsb",
+      "@tstromberg",
+      "@timothysc",
+      "@mtaufen",
+      "@rosti",
+      "@randomvariable",
+      "@fabriziopandini",
+      "@neolit123"
+    ],
+    "approvers": ["@timothysc", "@neolit123", "@fabriziopandini"],
+    "editor": "",
+    "creationDate": "2019-05-06",
+    "lastUpdated": "2019-05-29",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Kubeadm machine output\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Story 1](#story-1)\n    - [Story 2](#story-2)\n    - [Story 3](#story-3)\n    - [Story 4](#story-4)\n    - [Story 5](#story-5)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n    - [Details](#details)\n      - [A centralized printer system](#a-centralized-printer-system)\n      - [Decoupling commands from printing](#decoupling-commands-from-printing)\n    - [Notes](#notes)\n      - [Previous and related works](#previous-and-related-works)\n      - [Buffered vs unbuffered](#buffered-vs-unbuffered)\n      - [Parity with kubectl and versioned output](#parity-with-kubectl-and-versioned-output)\n      - [jq](#jq)\n      - [A friendly bootstrap token struct](#a-friendly-bootstrap-token-struct)\n      - [Go template functions](#go-template-functions)\n      - [Kubeadm init JSON output](#kubeadm-init-json-output)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Drawbacks](#drawbacks)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n**ACTION REQUIRED:** In order to merge code into a release, there must be an issue in [kubernetes/enhancements] referencing this KEP and targeting a release milestone **before [Enhancement Freeze](https://github.com/kubernetes/sig-release/tree/master/releases)\nof the targeted release**.\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e., [kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be updated for the enhancement to be released.\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nKubeadm should support structured output, such as JSON, YAML, or a [Go template](https://golang.org/pkg/text/template/).\n\n## Motivation\n\nWhile Kubernetes may be deployed manually, the de facto, if not de jure, means of turning up a Kubernetes cluster is kubeadm. Popular systems management software, such as Terraform, rely on kubeadm in order to deploy Kubernetes. Planned enhancements to the Cluster API project include a composable package for bootstrapping Kubernetes with kubeadm and cloud-init.\n\nWithout structured output, even the most seemingly innocuous changes could break Terraform, Cluster API, or other software that relies on the results of kubeadm.\n\n### Goals\n\n* Kubeadm should support structured output, including, but not limited to, the following output types:\n  * Buffered\n    * JSON\n    * YAML\n    * Go template\n  * Unbuffered\n    * Text (the current behavior)\n  * Please see [*Buffered vs Unbuffered*](#buffered-vs-unbuffered) for details on what both terms mean in the context of this KEP\n* Kubeadm should support the above output types for commands that include, but not necessarily limited to, the following:\n  * `alpha certs`\n  * `config images list`\n  * `init`\n  * `token create`\n  * `token list`\n  * `upgrade plan`\n  * `version`\n\n### Non-Goals\n\n* Explicit support for an unbuffered, *short* variant of *Text* output is not necessary at this time as the only command that currently supports this output type is `version`\n* [Parity with kubectl's `-o|--output` flag](#parity-with-kubectl-and-versioned-output)\n* [Versioned output](#parity-with-kubectl-and-versioned-output)\n* [Using the printers](#parity-with-kubectl-and-versioned-output) from the `kubernetes/cli-runtime` package\n\n## Proposal\n\n### User Stories\n\nThe examples in the user stories are predicated on:\n\n* [Awareness of `jq`](#jq)\n* [A friendly bootstrap token struct](#a-friendly-bootstrap-token-struct)\n* [Additional functions for Go templates](#go-template-functions)\n\n#### Story 1\n\nThe deployment of a multi-node Kubernetes cluster is automated by parsing [the JSON output](#kubeadm-init-json-output) of `kubeadm init` for the emitted `kubeadm join` command.\n\n1. Run `kubeadm init -o json | jq -r '\"kubeadm join \\(.node0) --token \\(.token.id) --discovery-token-ca-cert-hash \\(.caCrt)\"'`\n\n#### Story 2\n\nA script requires a list of token IDs.\n\n1. Run `kubeadm token list -o go-template='{{range .}}{{println .ID}}{{end}}'`\n\n#### Story 3\n\nA script returns a list of token IDs as they are discovered.\n\n1. Run `kubeadm token list --stream -o go-template='{{println .ID}}'`\n\n#### Story 4\n\nA script needs to process the IDs of all the non-expired tokens:\n\n1. Run `kubeadm token list -o json | jq '.[] | select(.expires | fromdate \u003e now) | .id'`\n\n#### Story 5\n\nA script needs to process the IDs of all the non-expired tokens as the tokens are discovered:\n\n1. Run `kubeadm token list --stream -o json | jq 'select(.expires | fromdate \u003e now) | .id'`\n\n### Implementation Details/Notes/Constraints\n\n#### Details\n\n##### A centralized printer system\nThe first design detail is related to how commands handle output...they don't. Or rather, they shouldn't. A command should perform an action and return a result, not print that result. Much like the package [`k8s.io/cli-runtime/pkg/printers`](https://github.com/kubernetes/cli-runtime), a package dedicated to printing will be introduced to kubeadm.\n\nWhether by virtue of an interface that describes a printer, or a single, exported function, the signature of a printer should be:\n\n```golang\nfunc Print(w io.Writer, format string, data interface{}) error\n```\n\nA very pseudo-codish implementation of the above signature might look something like this:\n\n```golang\nfunc Print(w io.Writer, format string, data interface{}) error {\n\t// If data is an io.Reader then copy the reader to the writer.\n\tif r, ok := data.(io.Reader); ok {\n\t\t_, err := io.Copy(w, r)\n\t\treturn err\n\t}\n\n\t// Handle pre-defined formats.\n\tswitch format {\n\tcase \"json\":\n\t\treturn json.NewEncoder(w).Encode(data)\n\tcase \"yaml\":\n\t\tbuf, err := yaml.Marshal(data)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t_, err := io.Copy(w, bytes.NewReader(buf))\n\t\treturn err\n\tcase \"text\":\n\t\t_, err := fmt.Fprintln(w, data)\n\t\treturn err\n\t}\n\n\t// Treat the format as a Go template.\n\ttpl, err := template.New(\"t\").Parse(format)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn tpl.Execute(w, data)\n}\n```\n\n##### Decoupling commands from printing\nIf a command is no longer in charge of rendering its output, how does a command submit its output to be rendered? There are several possibilities. Commands could:\n\n1. Receive a Printer object/function used to print results\n2. Send results to an exported, package-level function in the `printers` package\n3. Instantiate a new priner from the `printers` package\n4. Return a `chan interface{}` that receives the command's result(s). This channel can return an `io.Reader` or block until the command has buffered the output into a defined struct to be formatted by some Go template.\n\n#### Notes\n\n##### Previous and related works\n\n* Related\n  * https://github.com/kubernetes/kubeadm/issues/494\n  * https://github.com/kubernetes/kubeadm/issues/659\n  * https://github.com/kubernetes/kubeadm/issues/953\n  * https://github.com/kubernetes/kubeadm/issues/972\n  * https://github.com/kubernetes/kubeadm/issues/1454\n* Replaces\n  * https://github.com/kubernetes/kubernetes/pull/75894 ([design doc](https://docs.google.com/document/d/1YzFjb-lTW6HZDvxdG-pwYc4RbSdHvsJ5mZyywQY_QD0))\n\n##### Buffered vs unbuffered\n\nPlease note that *buffered* and *unbuffered* relates to individual objects emitted by a command, not the entirety of a command's output. For example, the command `kubeadm token list` may elect to format a list of tokens as JSON after all of the tokens have been discovered, but the same command might also print each token as they are discovered. The behavior will depend on the command and its flags.\n\nFor more clarificaton, please see [this example](https://play.golang.org/p/_CJLB7gdLZQ) that highlights how buffered printer output versus unbuffered printer output might behave in the context of this KEP.\n\n##### Parity with kubectl and versioned output\n\nParity with kubectl is defined as support for all of the output formats currently available to kubectl's `-o|--output` flag:\n\n  * `json`\n  * `yaml`\n  * `name`\n  * `template`\n  * `go-template`\n  * `go-template-file`\n  * `templatefile`\n  * `jsonpath`\n  * `jsonpath-file`\n\nThe natural path to such parity would be to use the same mechanism in kubeadm as used by kubectl, the API machinery package `k8s.io/cli-runtime/pkg/printers`. However, the printers require input objects of type `runtime.Object`.\n\nCreating new or converting existing kubeadm objects to Kubernetes API-style objects has the immediate effect of introducing versioned output to kubeadm. It's the shared opinion of the authors of this KEP that versioned output should be a non-goal. This does not indicate an opinion on the value of versioned output, but rather acknowledges that such a design decision requires a much broader discussion.\n\n##### jq\nThe program [`jq`](https://stedolan.github.io/jq/) is a performant, command-line solution for parsing and manipulating JSON.\n\n##### A friendly bootstrap token struct\n\n```golang\nstruct {\n\tID          string    `json:\"id\"`\n\tTTL         string    `json:\"ttl\"` // parseable by time.ParseDuration\n\tExpires     time.Time `json:\"expires\"`\n\tUsages      []string  `json:\"usages\"`\n\tDescription string    `json:\"description\"`\n}\n```\n\n##### Go template functions\n\nThe Go template into which objects are emitted will have a function map that includes the following functions:\n\n| Signature | Description |\n| `node0() string` | Returns the `addr:port` of the first node in a cluster |\n| `caCrt() string` | Returns the `sha256:hash` of the discovery token's CA certificate |\n| `join(list []string, sep string) string` | Calls `strings.Join(list, sep)` |\n\n##### Kubeadm init JSON output\n\nThe following JSON output is an example of running `kubeadm init -o json`:\n\n```json\n{\n  \"node0\": \"192.168.20.51:443\",\n  \"caCrt\": \"sha256:1f40ff4bd1b854fb4a5cf5d2f38267a5ce5f89e34d34b0f62bf335d74eef91a3\",\n  \"token\": {\n    \"id\":          \"5ndzuu.ngie1sxkgielfpb1\",\n    \"ttl\":         \"23h\",\n    \"expires\":     \"2019-05-08T18:58:07Z\",\n    \"usages\":      [\n      \"authentication\",\n      \"signing\"\n    ],\n    \"description\": \"The default bootstrap token generated by 'kubeadm init'.\",\n    \"extraGroups\": [\n      \"system:bootstrappers:kubeadm:default-node-token\"\n    ]\n  },\n  \"raw\": \"Rm9yIHRoZSBhY3R1YWwgb3V0cHV0IG9mIHRoZSAia3ViZWFkbSBpbml0IiBjb21tYW5kLCBwbGVhc2Ugc2VlIGh0dHBzOi8vZ2lzdC5naXRodWIuY29tL2FrdXR6LzdhNjg2ZGU1N2JmNDMzZjkyZjcxYjZmYjc3ZDRkOWJhI2ZpbGUta3ViZWFkbS1pbml0LW91dHB1dC1sb2c=\"\n}\n```\n\n### Risks and Mitigations\n\n**Risk**: Not including support for [versioned output](#parity-with-kubectl-and-versioned-output) in the stated [goals](#goals) could possibly result in an implied judgement of versioned output, when that's not the reasoning behind omitting it from this KEP.\n\n\n**Risk**: At first glance it may appear this KEP breaks compatibility with the existing output format for `kubeadm token list`, but that's not the case. \n\n*Mitigation*: Because the proposed printer design accepts an `io.Writer`, there is nothing to prevent the writer from being:\n\n```golang\nimport \"text/tabwriter\"\n...\ntw := tabwriter.New(os.Stdout, 0, 8, 0, '\\t', 0)\n```\n\nGiving `tw` to the printer along with the appropriate Go template ensures that support for tabular output is preserved.\n\n## Design Details\n\n### Test Plan\n\nThe test plan involves:\n\n* Matching actual output against expected output\n* An automated pipeline that connects `kubeadm init` to `kubeadm join` by parsing the structured output of the former to execute the latter\n\n### Graduation Criteria\n\nThis proposal targets *Alpha* support for structured kubeadm output in the release of Kubernetes 1.16.\n\n##### Alpha -\u003e Beta Graduation\n\n* The topic of versioned output is discussed thoroughly by the community and feedback is adapted with changes\n* The following commands implement structured output:\n  * `alpha certs`\n  * `config images list`\n  * `init`\n  * `token create`\n  * `token list`\n  * `upgrade plan`\n  * `version`\n* The feature is maintained by active contributors.\n* The feature is tested by the community and feedback is adapted with changes.\n* A test pipeline that takes the JSON output of `kubeadm init` in order to join a second node\n* Improved documentation\n\n##### Beta -\u003e GA Graduation\n\n* The feature is well tested and adapted by the community.\n* E2e test provide sufficient coverage.\n* Documentation is complete.\n\n### Upgrade / Downgrade Strategy\n\nNA\n\n### Version Skew Strategy\n\nNA\n\n## Implementation History\n\n* May 2019 (1.14) KEP was created. \n\n## Drawbacks \n\nNA\n"
+  },
+  {
+    "id": "5dfc6e1f5595a3a6cd0c5049dc44b2a8",
+    "title": "Advanced configurations with kubeadm (Kustomize)",
+    "authors": ["@fabriziopandini"],
+    "owningSig": "sig-cluster-lifecycle",
+    "participatingSigs": ["sig-cluster-lifecycle"],
+    "reviewers": ["@neolit123", "@rosti", "@ereslibre", "@detiber", "@vincepri", "@chuckha"],
+    "approvers": ["@timothysc", "@luxas"],
+    "editor": "@fabriziopandini",
+    "creationDate": "2019-07-22",
+    "lastUpdated": "2019-09-02",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Advanced configurations with kubeadm (Kustomize)\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Story 1](#story-1)\n    - [Story 2](#story-2)\n    - [Story 3](#story-3)\n  - [Implementation Details](#implementation-details)\n    - [Kustomize integration with kubeadm](#kustomize-integration-with-kubeadm)\n    - [Providing and storing Kustomize patches to kubeadm](#providing-and-storing-kustomize-patches-to-kubeadm)\n    - [Storing and retrieving Kustomize patches for kubeadm](#storing-and-retrieving-kustomize-patches-for-kubeadm)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Drawbacks](#drawbacks)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP\n- [x] KEP approvers have set the KEP status to `implementable`\n- [x] Design details are appropriately documented\n- [x] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [x] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nThis KEP is aimed at defining a new kubeadm feature that will allow users to bootstrap \na Kubernetes cluster with static pods customizations not supported by the Kubeadm component configuration.\n\n## Motivation\n\nKubeadm currently allows you to define a limited set of configuration options for a\nKubernetes cluster via the Kubeadm component configuration or the corresponding CLI flags.\n\nMore specifically the kubeadm component configuration provides an abstraction that allows to:\n\n1. To define configurations settings at cluster level using the `ClusterConfiguration`\n   config object \n2. To define a limited set of configurations at the node level using the\n   `NodeRegistrationOptions` object or the `localAPIEndpoint` object\n\nThis abstraction is well suited for the most common cluster configurations/use cases, but\nthere are other use cases that cannot be achieved with the kubeadm component configuration\nas of today. Some examples:\n\n- It is not possible to add sidecars e.g. for authorization web-hooks serving components.\n- It is not possible to set/alter timeouts for liveness probes in control plane components.\n\nThis KEP aims to introduce a new feature that will eneable users full control of static \npod manifest generated by Kubeadm at node level - vs the kubeadm component configuration\nthat allows mostly cluster-wide configurations on control-plane/etcd args only -.\n\nWith this new feature users should not be required anymore to manually alter static\npod manifests stored into `/etc/kubernetes/manifests` after kubeadm init/join.\n\n### Goals\n\nConsidering the complexity of this topic, this document is expected to be subject\nto some iterations. The goal of the current iteration is to:\n\n- Get initial approval on Summary and Motivation paragraphs\n- To identify a semantic for defining “advanced configurations” for control-plane/etcd\n  static pod manifests.\n- To define UX for passing “advanced configurations” to kubeadm init and to kubeadm join.\n- To define mechanics, limitations, and constraints for preserving “advanced configurations”\n  during cluster lifecycle and more specifically for supporting the kubeadm upgrade workflow.\n- To ensure the proper functioning of “advanced configurations” with kubeadm phases.\n- To clarify what is in the scope of kubeadm and what instead should be the responsibility\n  of the users/of higher-level tools in the stack like e.g. cluster API \n\n### Non-Goals\n\n- To provide any validation or guarantee about the security, conformance, \n  consistency, of “advanced configurations” for control-plane/etcd settings. \n  As it is already for `extraArgs` fields in the kubeadm component configuration or in the\n  Kubelet/KubeProxy component config, the responsibility of proper usage of those \n  advanced configuration options belongs to higher-level tools/users.\n- To deprecate the Kubeadm component configuration because:\n  - The component configuration provides an abstraction well suited for most common use\n    cases (that can be addressed with cluster-wide configurations on control-plane/etcd command line args only)\n  - The component configuration implicitly defines the main cluster variants the kubeadm team \n    is committed to support and monitor in the Kubernetes test grid.\n- To define how to manage “advanced configurations” when kubeadm and [`etcdadm`](https://github.com/kubernetes-sigs/etcdadm)\n  project will integrate. This will be defined in following iterations of this KEP.\n- To define how to manage “advanced configurations” for the addons \n  (this is postponed until kubeadm - [`addons`](https://github.com/kubernetes-sigs/addon-operators) \n  project integration).\n\n## Proposal\n\n### User Stories \n\n#### Story 1\nAs a cluster administrator, I want to add a sidecar to the kube-apiserver pod for running an\nauthorization web-hooks serving component.\n\n#### Story 2\nAs a cluster administrator, I want to set timeouts for the kube-apiserver liveness\nprobes for edge clusters.\n\n#### Story 3\nAs a cluster administrator, I want to upgrade my cluster preserving all the\n“advanced configuration” already in place\n\n### Implementation Details\n\nThis proposal explores as a first option for implementing Kubeadm \n“advanced configurations“ the usage of Kustomize; please refer to\n[Declarative application management in Kubernetes](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/architecture/declarative-application-management.md)\nand [Kustomize KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-cli/0008-kustomize.md)\nfor background information about Kustomize.\n\n#### Kustomize integration with kubeadm\n\nBy adopting Kustomize in the context of Kubeadm, this proposal assumes to:\n\n- Let kubeadm generate the static pod manifests as usual.\n- Use kubeadm generated artifacts as a starting point for applying patches\n  containing “advanced configurations”.\n\nThis has some implications:\n\n1. The higher-level tools/users have to express “advanced configurations” using\n   one of the two alternative techniques supported by Kustomize - the [strategic\n   merge patch](https://github.com/kubernetes-sigs/kustomize/blob/master/docs/glossary.md#patchstrategicmerge)\n   and the [JSON patch](https://github.com/kubernetes-sigs/kustomize/blob/master/docs/glossary.md#patchjson6902) -.\n2. The higher-level tools/users have to provide patches before running kubeadm;\n   this point is going to be further discussed in the following paragraphs.\n3. Kubeadm is responsible for coordinating the execution of Kustomize within the\n   init/join/upgrade workflows\n\nAdditionally, in order to simplify the first implementation of this KEP, this \nproposal is going to assume that Kustomize patches for kubeadm are always defined\nspecifically for the node where kubeadm is going to be executed.\n\nThis point could be reconsidered in the future, by e.g. introducing cluster-wide\npatches and/or patches for a subset of nodes.\n\nThe resulting workflow for using in kustomize will be the following\n\n1 - Create a folder with some patches, e.g.\n\n```\nmkdir kubeadm-patches\n\ncat \u003c\u003cEOF \u003e./kubeadm-patches/patch1.yaml\napiVersion: v1\nkind: Pod\nmetadata:\n  name: kube-apiserver\n  namespace: kube-system\n{your kube-apiserver patch here}\nEOF\n\ncat \u003c\u003cEOF \u003e./kubeadm-patches/patch2.yaml\napiVersion: v1\nkind: Pod\nmetadata:\n  name: kube-controller-manager\n  namespace: kube-system\n{your kube-controller-manager patch here}\nEOF\n```\n\nIn case higher-level tools/users are providing only strategic merge patches, like in the example above,\nit is not requested to take care of defining a `kustomization.yaml` file. In case the `kustomization.yaml` \nis missing kubeadm will create it on the fly using all the patches in the folder. \n\nIf instead higher-level tools/users are providing a `kustomization.yaml` file, kubeadm will use it.\nThis scenario allows e.g. usage of `patchesJson6902`/other kustomize features.\n\n\u003e In case a `kustomization.yaml` file exist, kubeadm ignores the `resource` value and replaces it with \n\u003e the static pod file manifest that should be kustomized by each phase.\n\n2 - Run kubeadm passing the folder where kustomization patches exist, e.g.\n\n```\nkubeadm init --experimental-kustomize kubeadm-patches/\nor \nkubeadm init phase control-plane ...  --experimental-kustomize kubeadm-patches/\n```\nor\n```\nkubeadm join --experimental-kustomize kubeadm-patches/\nor\nkubeadm join phase control-plane-prepare control-plane --experimental-kustomize kubeadm-patches/\n```\nor\n```\nkubeadm upgrade apply --experimental-kustomize kubeadm-patches/\n```\nor\n```\nkubeadm upgrade node --experimental-kustomize kubeadm-patches/\nor \nkubeadm upgrade node phase control-plane  --experimental-kustomize kubeadm-patches/\n```\n\nNB1. `--experimental-kustomize` is the proposed name for the flag to be renamed to --kustomize`\nwhen beta is reached. `-k` abbreviation can be reserved or even fully connected.\n\n#### Providing and storing Kustomize patches to kubeadm\n\nBefore kubeadm init, Kustomize patches should be eventually provided to kubeadm\nby higher-level tools/users; patches should be defined in a custom location \non the machine file system and this location could be passed to\nkubeadm init/join with a CLI flag. \n\nIn order to simplify the first implementation of this KEP, this proposal is assuming\nto use the same approach also for kubeadm join; this point could be reconsidered\nin the future, by e.g. defining a method for allowing higher-level tools/users to\ndefine Kustomize patches using a new CRD.\n\n#### Storing and retrieving Kustomize patches for kubeadm\n\nKustomize patches, should be preserved during the whole cluster lifecycle, mainly\nfor allowing kubeadm to preserve changes during the kubeadm upgrade workflow.\n\nIn order to simplify the first implementation of this KEP, this proposal is assuming\nthat Kustomize patches will remain stored in the custom location on the machine file system\nfor the necessary time, and that this location will be passed to kubeadm upgrade with a CLI \nflag; this point could be reconsidered in the future, by e.g. defining a method for\nallowing higher-level tools/users to define Kustomize patches using a new CRD.\n\n### Risks and Mitigations\n\n_Confusion between kubeadm component configuration and kustomize_\nKubeadm already offers a way to implement cluster settings, that is the kubeadm component\nconfiguration and component configs. Adding a new feature for supporting “advanced configurations”\ncan create confusion in the users.\n\nkubeadm maintainers should take care of making differences cristal clear in release notes\nand feature announcement:\n\n- The component configuration provides an abstraction well suited for most common use cases \n  that can be addressed with cluster-wide configurations on control-plane/etcd command line args only,\n  while “advanced configurations”/kustomize allows users full control of static \n  pod manifest generated by Kubeadm at node level.\n- The component configuration implicitly defines the main cluster variants the kubeadm team is\n  committed to support and monitor in the Kubernetes test grid, while instead higher-level\n  tools/user are responsible for the security, conformance, consistency, of \n  “advanced configurations” for control-plane/etcd static pod manifests. \n\n_Misleading expectations on the level of flexibility_\nIn order to provide guarantee about kubeadm respecting “advanced configurations” during\ninit, join, upgrades or single-phase execution, it is necessary to define some trade-offs\naround _what_ can be customized and _how_.\n\nEven if the proposed solution is based on the user feedback/issues, the kubeadm maintainers\nwant to be sure the implementation is providing the expected level of flexibility and, in\norder to ensure that, we will wait for at least one K8s release cycle for the users to provide \nfeedback before moving forward in graduating the feature to beta. \n\nSimilarly, the kubeadm maintainers should work with [`etcdadm`](https://github.com/kubernetes-sigs/etcdadm)\nproject and [`addons`](https://github.com/kubernetes-sigs/addon-operators) project\nto ensure a consistent approach across different components.\n\n_Component version change during upgrades_\nStatic pod manifests are managed by kubeadm, while “advanced configurations”/kustomize patches\nwill be managed by users.\n\nIt might happen that a static pod manifest during upgrades is changed by kubeadm in a way that\nsome patches will not cleanly apply anymore. \n\nThe kubeadm maintainers will work on release notes to make potential breaking changes more visible;\nadditionally, upgrade instructions will be updated adding the recommendation to --dry-run and check\nexpected changes before upgrades.\n\n_Kustomize errors during kubeadm workflows_\nWhen executing “advanced configurations”/kustomize patches within kubeadm workflows, we are introducing\nan external element that can potentially generate errors during commands execution.\n\nError management should be adapted to this new possible risk, ensuring that the node remains in a\nconsistent state in case of errors.\n\n## Design Details\n\n### Test Plan\n\nAdd at least one periodic E2E test job exercising  “advanced configurations” during init,\njoin and upgrades.\n\nPlease note that, in accordance with the split of responsibilities defined in the previous paragraphs, \nthe new E2E test will focus _only_ on the mechanics of applying “advanced configurations”/kustomize \npatches, not on the possible combination of patches and nor on the security, conformance, consistency,\nof the resulting Kubernetes cluster.\n\n### Graduation Criteria\n\nThis proposal in its initial version covers only the creation of a new alpha feature.\nGraduation criteria will be defined in the following iterations on this proposal and\nconsider user feedback as well.\n\n### Upgrade / Downgrade Strategy\n\nAs stated in goals, kubeadm will preserve “advanced configurations” during upgrades,\nand more specifically, it will re-apply patches after each upgrade.\n\nDowngrades are not supported by kubeadm, and not considered by this proposal.\n\n### Version Skew Strategy\n\nThis proposal does not impact kubeadm compliance with official K8s version skew policy;\nhigher-level tools/user are responsible for the security, conformance, consistency of\n“advanced configurations” that can impact or the aforementioned point.\n\n## Implementation History\n\nTuesday, July 30, 2019\n- the `Summary` and `Motivation` sections being merged signaling SIG acceptance\n- the `Proposal` section being merged signaling agreement on a proposed design\n- the date implementation started\n\nv1.16\n- first implementation (alpha)\n\n## Drawbacks\n\nKubeadm already offers a way to implement cluster settings, that is the kubeadm API\nand the support for providing component configs. Adding a new feature for supporting\n“advanced configurations” can create confusion in the users.\n\nSee risks and mitigations.\n\n## Alternatives\n\nThere are many alternatives to “Kustomize” in the ecosystem; see [Declarative application management in Kubernetes](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/architecture/declarative-application-management.md).\n\nWhile there is great value in several different approaches “Kustomize” was selected as\nthe first choice for this proposal because it already has first-class supported in\nkubectl (starting from v1.14).\n"
+  },
+  {
+    "id": "0f1f13008d7677918065ece1d2b04dcb",
+    "title": "Kubeadm operator",
+    "authors": ["@fabriziopandini"],
+    "owningSig": "sig-cluster-lifecycle",
+    "participatingSigs": ["sig-cluster-lifecycle"],
+    "reviewers": [
+      "@neolit123",
+      "@rosti",
+      "@ereslibre",
+      "@detiber",
+      "@vincepri",
+      "@yastij",
+      "@chuckha"
+    ],
+    "approvers": ["@timothysc", "@luxas"],
+    "editor": "@fabriziopandini",
+    "creationDate": "2019-09-16",
+    "lastUpdated": "2019-09-28",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Kubeadm operator\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Story 1](#story-1)\n    - [Story 2](#story-2)\n    - [Story 3](#story-3)\n    - [Story 4](#story-4)\n    - [Story 5](#story-5)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Drawbacks](#drawbacks)\n- [Alternatives](#alternatives)\n- [Infrastructure Needed](#infrastructure-needed)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location\n      in kubernetes/enhancements, not the initial KEP PR)\n- [x] KEP approvers have set the KEP status to `implementable`\n- [x] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings,\n      relevant PRs/issues, release notes\n\n## Summary\n\nKubeadm operator would like to enable declarative control of kubeadm workflows, automating the execution and the\norchestration of such tasks across existing nodes in a cluster.\n\n## Motivation\n\nKubeadm binary can execute operations only on the machine where it is running e.g. it is not possible to execute\noperations on other nodes, to copy files across nodes, etc.\n\nAs a consequence, most of the kubeadm workflows, like [kubeadm upgrade](https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-upgrade/),\nconsists of a complex sequence of tasks that should be manually executed and orchestrated across all the existing nodes\nin the cluster.\n\nSuch a user experience is not ideal due to the error-prone nature of humans running commands. The manual approach\ncan be considered a blocker for implementing more complex workflows such as rotating certificate authorities,\nmodifying the settings of an existing cluster or any task that requires coordination of more than one Kubernetes node.\n\nThis KEP aims to address such problems by applying the operator pattern to kubeadm workflows. \n\n### Goals\n\n- To allow declarative control of kubeadm operations that lead to \"in place\" mutations[1] of kubeadm generated\n  artifacts. More specifically kubeadm artifacts are static pod manifests, certificates, kubeconfig file,\n  bootstrap token, kubeadm generated configmap and secrets, while this proposal initially includes the following\n  kubeadm workflows:\n  - kubeadm upgrade\n  - certificate renewal\n  - certificate authority rotation (NEW)\n  - change configuration in an existing cluster (NEW) \n\n\u003e [1] Please note that we are referring to \"in place\" mutations of kubeadm generated artifacts in order to highlight\n\u003e the difference between the kubeadm operator and other SCL projects like [Cluster API](https://cluster-api.sigs.k8s.io/),\n\u003e which instead assume nodes and underlying machines are immutable.\n\nConsidering the complexity of this topic, this document is expected to be subject to some iterations.\nThe goal of the current iteration is to:\n\n- Get initial approval on Summary and Motivation paragraphs\n- To identify a semantic for defining “Operations” to be performed by the kubeadm-operator.\n- To define how the kubeadm-operator should manage kubeadm workflows.\n- To define how the users should interact with the kubeadm-operator, including also observability and error handling.\n- To define how the kubeadm-operator should be deployed in a Kubernetes cluster.\n\n### Non-Goals\n\n- To provide or manage any infrastructure elements such as underlying machines, load balancers, storage, etc.\n- To manage and automate the kubeadm init and join workflows.\n- To manage any artifact *not* generated by kubeadm. The only exception are the kubelet, kubeadm, andkubectl\n  binaries which are considered in-scope as required by the upgrade workflow.\n- To replace kubeadm \"raw\" workflows. The user will always be able to run kubeadm workflows in isolation in \n  manual fashion.\n\n## Proposal\n\n### User Stories\n\n#### Story 1\n\nAs a Kubernetes operator, I would like to be able to declaratively control upgrades in a systematic fashion.\n\n#### Story 2\n\nAs a Kubernetes operator, I would like to be able to declaratively control certificate renewal in a systematic fashion.\n\n#### Story 3\n\nAs a Kubernetes operator, I would like to be able to declaratively control changes of the current cluster's settings\nin a systematic fashion.\n\n#### Story 4\n\nAs a Kubernetes operator, I would like to be able to declaratively rotate my certificate authorities.\n\n#### Story 5 \nAs a Kubernetes operator, I would like to control whether nodes are cordoned and drained for tasks or if they are\nperformed without disruption to the workloads on the node, referred to as a \"hot\" update.\n\n### Implementation Details/Notes/Constraints\n\nAt his core the kubeadm operator is a controller that watches kubeadm ClusterConfiguration object, and according\nto its changes, to trigger the required operations to reconcile the desired state and the current state, where\neach operations is one of the supported kubeadm workflow.\n\n![](20190916-kubeadm-operator.png)\n\nIn order to break down the the implementation of the kubeadm operator into smaller, actionable items, the current\niteration of this KEP focuses on a more limited scope, that is the orchestration of the operations that\nthe kubeadm operator is going to execute for reconcyling the currente state and the desired state.\n\nA POC will be executed in order to:\n\n- Identify a semantic for defining “Operations” to be performed by the kubeadm-operator.\n- Define how the kubeadm-operator should manage kubeadm workflows.\n- Define how the users should interact with the kubeadm-operator, including also observability and error handling.\n- Define how the kubeadm-operator should be deployed in a Kubernetes cluster.\n\n### Risks and Mitigations\n\nTBD\n\n## Design Details\n\n### Test Plan\n\nTBD\n\n### Graduation Criteria\n\nTBD\n\n### Upgrade / Downgrade Strategy\n\nTBD\n\n### Version Skew Strategy\n\nTBD\n\n## Implementation History\n\n- the `Summary` and `Motivation` sections being merged signaling SIG acceptance\n\n## Drawbacks\n\nTBD\n\n## Alternatives\n\n[1] To NOT implement the kubeadm operator, and let the user automate workflows/orchestration of kubeadm\nactions across nodes with other tools.\n\n## Infrastructure Needed\n\nTBD\n"
+  },
+  {
+    "id": "0df0654baafb4725cc8e66b40fe74c06",
+    "title": "kubeadm component config management",
+    "authors": ["@rosti"],
+    "owningSig": "sig-cluster-lifecycle",
+    "participatingSigs": ["sig-cluster-lifecycle"],
+    "reviewers": ["@fabriziopandini", "@neolit123", "@ereslibre", "@yastij"],
+    "approvers": ["@timothysc"],
+    "editor": "@rosti",
+    "creationDate": "2019-09-25",
+    "lastUpdated": "2019-09-25",
+    "status": "implementable",
+    "seeAlso": [
+      "/keps/sig-cluster-lifecycle/kubeadm/0023-kubeadm-config.md",
+      "/keps/sig-cluster-lifecycle/kubeadm/20190722-Advanced-configurations-with-kubeadm-(Kustomize).md"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# kubeadm component config management\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Stop Component Config defaulting](#stop-component-config-defaulting)\n  - [Delegate config validation to the components](#delegate-config-validation-to-the-components)\n  - [Kubernetes Core vs AddOns Component Configs](#kubernetes-core-vs-addons-component-configs)\n    - [AddOn Component Config management](#addon-component-config-management)\n    - [Kubernetes Core Component Config management](#kubernetes-core-component-config-management)\n      - [Generated Kubernetes Core Component Config management](#generated-kubernetes-core-component-config-management)\n      - [User supplied Kubernetes Core Component Config management](#user-supplied-kubernetes-core-component-config-management)\n      - [Allow Kustomize use over core Kubernetes component configs](#allow-kustomize-use-over-core-kubernetes-component-configs)\n  - [Stop using component config internal types](#stop-using-component-config-internal-types)\n  - [(OPTIONAL) “kubeadm upgrade plan” changes](#optional-kubeadm-upgrade-plan-changes)\n  - [Risks and Mitigations](#risks-and-mitigations)\n    - [Users will have to manually migrate more Kubernetes core component configs than before](#users-will-have-to-manually-migrate-more-kubernetes-core-component-configs-than-before)\n    - [Users will have to manually migrate more AddOn component configs than before](#users-will-have-to-manually-migrate-more-addon-component-configs-than-before)\n    - [Users will loose track of what needs manual migration and what not](#users-will-loose-track-of-what-needs-manual-migration-and-what-not)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n  - [09/25/2019](#09252019)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [X] KEP approvers have set the KEP status to `implementable`\n- [X] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nThis document outlines a model by which kubeadm will manage component configs other than its own.\n\n## Motivation\n\nFor a long time now, kubeadm has been using component configs for kubelet and kube-proxy. These configs are usually generated by kubeadm by using a default component config as a basis and adding setup specific details (either opinionated by kubeadm itself, or supplied via some of the kubeadm’s config objects).  \nThere is support for users to supply their own component configs along with the kubeadm specific config objects upon kubeadm init. Kubeadm validates, defaults and stores these config on the cluster in the form of config maps. It also supplies these configs in one way or another to the target component itself.  \nThere is also a desire and code has been historically structured in a way, to allow kubeadm to convert component config formats upon upgrade, should the need for that arise.\n\nHence, the following problems have been observed:\n- Internal types have been used to allow for defaulting, validating and migrating component configs. This cannot continue if kubeadm is to move out of the kubernetes/kubernetes repository.\n- Defaulting the configs, that are generated by kubeadm or supplied by users, populates additional fields with default values. This makes it difficult to determine in retrospect, what fields were set on purpose with their default values, and what fields were simply automatically filled in by the defaulting code.\n- Providing support for validation by vendoring component specific code can lead to false-positive and missed errors. This can be caused by the deviation between the imported code by kubeadm and the actual component.\n- Migrating between config versions also requires code to be vendored in or forked from the components. This can also lead to difficult to diagnose errors and changed behaviour in unexpected ways.\n- Not distinguishing between kubeadm generated and user supplied component configs, necessitates the requirement for config migration. Kubeadm generated configs that are not modified by users directly in the config map, can be fully regenerated upon config format change. In all other cases kubeadm must convert the config to preserve user changes.\n\nThese observations along with the emergence of kubeadm phases, kustomize support and work around addon operators allows us to have the following future goals.\n\n### Goals\n\n- Stop vendoring internal types and validation, defaulting and conversion code.\n- Stop defaulting component configs.\n- Outsource validation to the component binaries themselves.\n- Distinguish between user supplied and kubeadm generated and unmodified component configs.\n- Eradicate the need to convert configs.\n- Use more community wide solutions, like Kustomize, kubeadm phases and addon operators.\n- Provide for as clean as possible upgrade path for existing clusters.\n\n### Non-Goals\n\n- Change the handling of kubeadm’s own config types in any way.\n- Cover in any way the migration of command line flags to component configs.\n- Define strategies for config map naming and backup.\n\n## Proposal\n\n### Stop Component Config defaulting\n\nComponent config defaulting has a number of issues:\n- It’s difficult to tell in retrospect if a setting value was intended or simply defaulted.\n- If a default value changes a defaulted setting will stop us from using the new value.\n- Defaulting bloats the resulting config, thus making it hard for users to migrate it upon a version change.\n\nHence, kubeadm should stop defaulting component configs and leave this to components themselves.\n\n### Delegate config validation to the components\n\nCurrently, kubeadm vendors in pieces of Go code from the components to be able to perform component config validation. This is a bad practice as it bloats the kubeadm dependencies and deviation between the vendored code and the components can lead to inconsistencies.  \nTo solve this, validation is going to be performed by executing a container or the binary of the component to be used with an appropriate command to verify the supplied config.  \nIf a target component does not have any means to verify its config, then kubeadm won’t do any validation for the time being. It will instead display a warning to the user denoting that certain component’s config is unvalidated.\n\n### Kubernetes Core vs AddOns Component Configs\n\nCurrently kubeadm maintains the component configs of both kubelet and kube-proxy. These are components that are maintained in very different ways.\n\nKubelet, along with API Server, Scheduler, Controller Manager and etcd, are deemed essential Kubernetes components. They are deployed and managed locally by kubeadm, using systemd services (kubelet) or static pods on the local kubelet (API Server, Scheduler, etc.). Hence, their configs are usually consumed by the components in the form of local files.\n\nOn the other hand, kube-proxy and CoreDNS/kube-dns are addons, that are managed as standard Kubernetes deployments. Hence their configs are usually accessed via config maps.\n\nTherefore, two different config lifecycles are distinguishable here - one for Kubernetes core components and one for cluster addons.\n\nThe defaulting and validation changes, proposed in the previous paragraphs, are going to be applied to both Kubernetes core and AddOn component config types.\n\n#### AddOn Component Config management\n\nAddOn component configs are always stored in config maps. They can be supplied in their entirety by users along with the kubeadm config during init or can be generated by kubeadm itself.\nIf, upon upgrade, kubeadm detects an obsolete config version, it will bail out with an error message, asking users to upgrade the addon config by hand.\n\nThe ultimate goal for addon component configs, is for them to be eventually managed in their entirety by cluster addon operators, instead of kubeadm.\n\n#### Kubernetes Core Component Config management\n\nLike in the addon case, Kubernetes core component configs can be specified by users in their entirety along with the kubeadm config during init or in a config map. On the other hand, if those configs are not user supplied, they are generated by kubeadm.\nUser supplied configs need to be stored in config maps and have to be manually converted by users upon upgrade, while kubeadm generated ones need neither of these things.  \nAs most users would choose kubeadm to generate and manage Kubernetes core component configs itself, it is a viable proposition to treat generated and user supplied configs differently.\n\n##### Generated Kubernetes Core Component Config management\n\nWhenever kubeadm does not find component configs, in its config file or in a set of well known config maps, it will opt in for generation.  \nGenerated core configs won’t be stored in config maps. Instead, they will always be regenerated in their entirety. This will allow seamless component config version upgrades when users have delegated config management responsibility to kubeadm.\n\n##### User supplied Kubernetes Core Component Config management\n\nAs user changes need to be maintained, user supplied configs are stored config maps.  \nHowever, in those cases kubeadm is not responsible for config migration upon upgrade. If the new component version is unable to consume the user supplied config, kubeadm is going to bail out with an error message, prompting the user to perform a manual config migration beforehand. This is a behavior similar to that for addon configs.\n\n##### Allow Kustomize use over core Kubernetes component configs\n\nKubeadm shall extend its current Kustomize support to include patching of core Kubernetes component configs. Currently, this will include only kubelet’s component config. However, in the future this might extend to the API Server, Scheduler and Controller Manager as well.  \nMost notably, this feature is not going to be applied to kube-proxy’s config by kubeadm as this is an addon component config and the responsibility for this is going to be with the corresponding addon operator.\n\nKustomize patches will be applied on both user supplied and kubeadm generated core component configs, just before they are stored in the local files from which the config will be consumed by the components. In the case of user supplied core component configs, neither the Kustomize patches, nor the result after applying them is going to be stored on the cluster. The config maps are going to contain only the vanilla user supplied component configs.\n\nIf a user has provided Kustomize patches for a config version, that is different from the current one, kubeadm is going to bail out with an error, prompting the user to either delete the old patches, or to upgrade them to the new version. This is best implemented using a kubeadm pre-flight check.\n\nUX wise, this feature is going to be reusing the existing kubeadm options that were introduced with the implementation of the kubeadm advanced configurations KEP. This allows for kustomizations to be applied to component configs on a per-node basis during init, join and upgrade just before the config is supposed to be stored on the local node file.\n\n### Stop using component config internal types\n\nAs kubeadm will no longer default, validate or migrate component configs inside its code base, the need to use component config internal types is no longer needed. In that case the usage of internal types can be removed. This would also allow us to reduce the number of imported packages and to drop one of the last dependencies on the kubernetes/kubernetes code base.  \nThe switch is going to be performed by replacing each internal type with a corresponding versioned type. This, by itself, is not going to have any impact on users.\n\n### (OPTIONAL) “kubeadm upgrade plan” changes\nAs supported component configs are expected to increase in number in the future and some of them might require manual user migration prior to upgrade, an ahead of time notice about what actions are required is a nice thing to have.  \nHence, the “kubeadm upgrade plan” command is going to provide information on the component configs. In particular, if they are managed by kubeadm or are user supplied, and if they need manual upgrade or not.\n\n### Risks and Mitigations\n\n#### Users will have to manually migrate more Kubernetes core component configs than before\n\nIn general, most of the users are expected to change only a few fields in the core Kubernetes component configs.\nThus, such configs are better managed by maintaining a Kustomize patch set, rather than a full blown component config.  \nWith a proper documentation in place, users will be encouraged to opt in for more kubeadm generated configs and Kustomize patch sets, when necessary.\n\n#### Users will have to manually migrate more AddOn component configs than before\n\nFor user supplied AddOn configs, that have changed config versions, this might be somewhat true.  \nIn the long run, addon operators are expected to manage component configs.\n\n#### Users will loose track of what needs manual migration and what not\n\nThe proposed changes to the \"kubeadm upgrade plan\" command will mitigate this risk, as kubeadm users are required to run that command prior to performing an actual upgrade.\n\n## Design Details\n\n### Test Plan\n\nIn addition to unit tests, extensive E2E tests should be created to verify various aspects if the implementation.  \nMost notably E2E tests should be written to verify different aspects of:\n- AddOn and Kubernetes core component configs\n- Configs supplied by users or generated by kubeadm\n- behavior during kubeadm init and upgrade\n\n### Graduation Criteria\n\nImplementing the accepted proposal in full with sufficient test coverage is the bare minimum for graduation to beta.\nHowever, as this is only an initial version of the proposal and further changes are expected over time. It's likely that the graduation criteria will change in the course of some alpha iterations.\n\n### Upgrade / Downgrade Strategy\n\nUpon upgrade to the new component config strategy, kubeadm users can do one of the following:\n\n- Delete old config maps and allow kubeadm to regenerate the configs.\n- Keep old configs untouched, but refuse an upgrade if an issue is discovered (e.g. old config format is no longer accepted by the component).\n\n### Version Skew Strategy\n\nN/A\n\n## Implementation History\n\n### 09/25/2019\n- Document created with Summary, Motivation, Proposal and other sections\n"
+  },
+  {
+    "id": "5e27d0e4f30190b2718016240fb9de07",
+    "title": "Remove ClusterStatus from kubeadm-config",
+    "authors": ["@fabriziopandini"],
+    "owningSig": "sig-cluster-lifecycle",
+    "participatingSigs": ["sig-cluster-lifecycle"],
+    "reviewers": ["@neolit123", "@rosti", "@ereslibre", "@ncdc"],
+    "approvers": ["@timothysc"],
+    "editor": "@fabriziopandini",
+    "creationDate": "2019-11-25",
+    "lastUpdated": "2019-11-25",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Remove ClusterStatus from kubeadm-config\n\n## Table of Contents\n\n\u003c!-- TOC --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n\u003c!-- /TOC --\u003e\n\n## Release Signoff Checklist\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [x] KEP approvers have set the KEP status to `implementable`\n- [x] Design details are appropriately documented\n- [x] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [x] Graduation criteria is in place\n- [x] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nThis KEP is proposing a new mode for tracking the list of the API endpoints in a cluster, thus allowing to remove the  `ClusterStatus` entry in the `kubeadm-config` ConfigMap and solve the problems that arise when, for any reasons, such entry does not reflect anymore the real status of the cluster.\n\n## Motivation\n\nIn order to manage HA cluster properly, kubeadm requires to have access to the list of API endpoints in a cluster.\n\nCurrently this feature is implemented by adding an entry in the list of API endpoints that is stored in the `ClusterStatus` entry of the `kubeadm-config` ConfigMap.\n\nThere are well known problem related to the management of this list e.g. when a control-plane node dies or is deleted without invoking `kubeadm reset`, the list gets stale and the user is required to manually cleanup the list in order to avoid any kubeadm operation that relies on such list might incur into errors.\n\nThis KEP is going to propose a different mode for tracking the list of the API endpoints in a cluster, based on the inspection of the current Pods.\n\nThis approach does not require the maintenance of a separated list, and implicitly always reflect the current status of the cluster.\n\nThis allows to remove the `ClusterStatus` entry in the `kubeadm-config` ConfigMap and to clean-up all the related code in `kubeadm init`, `kubeadm join` and `kubeadm reset`\n\n### Goals\n\n- To introduce a new method for tracking the list of API endpoints in a cluster\n- To allow removal of the `ClusterStatus` entry in the `kubeadm-config` ConfigMap and clean-up of the related code.\n\n### Non-Goals\n\n- To change any user facing behavior in `kubeadm`.\n  However, considering that we are going to remove the `ClusterStatus` in the `kubeadm-config` ConfigMap and potentially some user/higher level tools can rely on it, we are going issue and \"Action Required\" warning in the release note an respect the Beta API deprecation policy before actual deletion (9 months/3 release).\n\n## Proposal\n\n### Implementation Details/Notes/Constraints [optional]\n\nAs of today, the `ClusterStatus` entry in the `kubeadm-config` ConfigMap contains a map that stores the `LocalAPIEndpoint` for each control-plane node.\n\nThe `LocalAPIEndpoint` primary usage is for the `advertise-address`:`bindPort` flag in the `kube-apiserver` pod. Having this value in a flag is not ideal for the purpose of this proposal, so, we are going to echo the same value into a new annotation in the `kube-apiserver` pod manifest.\nThe annotation will be named `kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint` and the value will be `advertise-address`:`bindPort`.\n\nOnce the annotation will be in place, it will be possible to easily retrieve the local advertise address for each control plane node by querying the corresponding `kube-apiserver` pod.\n\nThe `LocalAPIEndpoint` is also used in the stacked `etcd` pod manifest for composing the `peer-urls` and the `client-urls`; the latter is used by kubeadm when accessing etcd in an existing cluster, e.g. when doing `join --control-plane`.\n\nWe are going to echo the `client-urls` value into a new annotation named `kubeadm.kubernetes.io/etcd.advertise-client-urls`. Once the annotation will be in place, it will be possible to easily retrieve the etcd client urls by querying the `etcd` pods.\n\n### Risks and Mitigations\n\nR. The list of API endpoints in a cluster is crucial to all the kubeadm workflows\nM. The proper function of those workflows and of the underlying codes is already covered by E2E tests; on top of that, we are going to try to implement this change at the beginning of the v1.18 cycle, thus ensuring as much\ntest cycles as possible.\n\nR. We are relying on the kubelet signal to check if a static pod goes down.\nM. The fact that there is a long-running agent doing this check is already an improvement vs current state; However, during the implementation, we should try to make this smarter as possible, e.g checking the pod status or any type of feedback the kubelet provides.\n\n## Design Details\n\n### Test Plan\n\nAll the affected behaviors are already covered by existing E2E test; however, we should consider if to add new destructive tests in order to exercise properly all the possible conditions.\n\nAdditional unit test are required only for the new function implementing the inspection of the current Pods.\n\n### Graduation Criteria\n\nAlpha - Does not apply (this is part of how kubeadm handles it's configuration API, and currently this is already in Beta).\nBeta - Initial status\nGA - This will graduate as part of the Kubeadm Configuration API.\n\n### Upgrade / Downgrade Strategy\n\nDuring upgrades:\n\n- The new annotations `kubeadm.kubernetes.io/kube-apiserver.advertise-address` and `kubeadm.kubernetes.io/etcd.advertise-client-urls` will be generated during the upgrade of the static pod manifests.\n- The `ClusterStatus` entry will be cleaned up during the upgrade of the `kubeadm-config` ConfigMap.\n\nDowngrade are not supported by kubeadm.\n\n### Version Skew Strategy\n\nNA\n\n## Implementation History\n\n- the `Summary` and `Motivation` sections being merged signaling SIG acceptance\n- the `Proposal` section being merged signaling agreement on a proposed design\n- the date implementation started\n"
+  },
+  {
+    "id": "879eaaf243523af70ca78045ae0d5692",
+    "title": "Moving ComponentConfig API types to staging repos",
+    "authors": ["@luxas", "@sttts"],
+    "owningSig": "sig-cluster-lifecycle",
+    "participatingSigs": [
+      "sig-api-machinery",
+      "sig-node",
+      "sig-network",
+      "sig-scheduling",
+      "sig-cloud-provider"
+    ],
+    "reviewers": ["@thockin", "@liggitt", "@wojtek-t", "@stewart-yu", "@dixudx"],
+    "approvers": ["@thockin", "@jbeda", "@deads2k"],
+    "editor": "@luxas",
+    "creationDate": "2018-07-07",
+    "lastUpdated": "2018-08-10",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Moving ComponentConfig API types to staging repos\n\n**How we can start supporting reading versioned configuration for all our components after a code move for ComponentConfig to staging**\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n  - [The current state of the world](#the-current-state-of-the-world)\n    - [Current kubelet](#current-kubelet)\n    - [Current kube-proxy](#current-kube-proxy)\n    - [Current kube-scheduler](#current-kube-scheduler)\n    - [Current kube-controller-manager](#current-kube-controller-manager)\n    - [Current kube-apiserver](#current-kube-apiserver)\n    - [Current cloud-controller-manager](#current-cloud-controller-manager)\n  - [Goals](#goals)\n  - [Non-goals](#non-goals)\n  - [Related proposals and further references](#related-proposals-and-further-references)\n- [Proposal](#proposal)\n  - [Migration strategy per component or k8s.io repo](#migration-strategy-per-component-or-k8sio-repo)\n    - [k8s.io/apimachinery changes](#k8sioapimachinery-changes)\n    - [k8s.io/apiserver changes](#k8sioapiserver-changes)\n    - [kubelet changes](#kubelet-changes)\n    - [kube-proxy changes](#kube-proxy-changes)\n    - [kube-scheduler changes](#kube-scheduler-changes)\n    - [k8s.io/controller-manager changes](#k8siocontroller-manager-changes)\n    - [kube-controller-manager changes](#kube-controller-manager-changes)\n    - [cloud-controller-manager changes](#cloud-controller-manager-changes)\n    - [kube-apiserver changes](#kube-apiserver-changes)\n  - [Vendoring](#vendoring)\n    - [Vgo](#vgo)\n    - [Dep](#dep)\n  - [Timeframe and Implementation Order](#timeframe-and-implementation-order)\n  - [OWNERS files for new packages and repos](#owners-files-for-new-packages-and-repos)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nCurrently all ComponentConfiguration API types are in the core Kubernetes repo. This makes them practically inaccessible for any third-party tool. With more and more generated code being removed from the core Kubernetes repo, vendoring gets even more complicated. Last but not least, efforts to move out kubeadm from the core repo are blocked by this.\n\nThis KEP is about creating new staging repos, `k8s.io/{component}`, which will host the external\ntypes of the core components’ ComponentConfig in a top-level `config/` package. Internal types will *eventually* be stored in\n`k8s.io/{component}/pkg/apis/config` (but a non-goal for this KEP). Shared types will go to `k8s.io/{apimachinery,apiserver,controller-manager}/pkg/apis/config`.\n\n### The current state of the world\n\n#### Current kubelet\n\n* **Package**: [k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig](https://github.com/kubernetes/kubernetes/blob/release-1.11/pkg/kubelet/apis/kubeletconfig/types.go)\n* **GroupVersionKind:** `kubelet.config.k8s.io/v1beta.KubeletConfiguration`\n* **Supports** reading **config from file** with **flag precedence**, **well-tested**.\n\n#### Current kube-proxy\n\n* **Package**: [k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig](https://github.com/kubernetes/kubernetes/blob/release-1.11/pkg/proxy/apis/kubeproxyconfig/types.go)\n* **GroupVersionKind**: `kubeproxy.config.k8s.io/v1alpha1.KubeProxyConfiguration`\n* **Supports** reading **config from file**, **without flag precedence**, **not tested**.\n* This API group has its own copy of `ClientConnectionConfiguration` instead of a shared type.\n\n#### Current kube-scheduler\n\n* **Package**: [k8s.io/kubernetes/pkg/apis/componentconfig](https://github.com/kubernetes/kubernetes/blob/release-1.11/pkg/apis/componentconfig/types.go)\n* **GroupVersionKind**: `componentconfig/v1alpha1.KubeSchedulerConfiguration`\n* **Supports** reading **config from file**, **without flag precedence**, **not tested**\n* This API group has its own copies of `ClientConnectionConfiguration` \u0026 `LeaderElectionConfiguration` instead of shared types.\n\n#### Current kube-controller-manager\n\n* **Package**: [k8s.io/kubernetes/pkg/apis/componentconfig](https://github.com/kubernetes/kubernetes/blob/release-1.11/pkg/apis/componentconfig/types.go)\n* **GroupVersionKind**: `componentconfig/v1alpha1.KubeControllerManagerConfiguration`\n* **No support for config from file**\n* This API group has its own copies of `ClientConnectionConfiguration` \u0026 `LeaderElectionConfiguration` instead of shared types.\n\n#### Current kube-apiserver\n\n* **Doesn’t expose component configuration anywhere**\n* **No support for config from file**\n* The most similar thing to componentconfig for the API server is the `ServerRunOptions` struct in\n  [k8s.io/kubernetes/cmd/kube-apiserver/app/options/options.go](https://github.com/kubernetes/kubernetes/blob/release-1.11/cmd/kube-apiserver/app/options/options.go)\n\n#### Current cloud-controller-manager\n\n* **Package**: [k8s.io/kubernetes/pkg/apis/componentconfig](https://github.com/kubernetes/kubernetes/blob/release-1.11/pkg/apis/componentconfig/types.go)\n* **GroupVersionKind**: `componentconfig/v1alpha1.CloudControllerManagerConfiguration`\n* **No support for config from file**\n* This API group has its own copies of `ClientConnectionConfiguration` \u0026 `LeaderElectionConfiguration` instead of shared types.\n\n### Goals\n\n* Find a home for the ComponentConfig API types, hosted as a staging repo in the \"core\" repo that is kubernetes/kubernetes\n* Make ComponentConfig API types consumable from *projects outside of kube* and from different parts of kube itself\n    * Resolve dependencies from the external ComponentConfig API types so that everything can depend on them\n    * The only dependency of the ComponentConfig API types should be `k8s.io/apimachinery`\n* Split internal types from versioned types\n* Remove the monolithic `componentconfig/v1alpha1` API group\n* Enable the staging bot so that a `[https://github.com/kubernetes/](https://github.com/kubernetes/){component}`\n  (imported as `k8s.io/{component}`) repos are published regularly.\n* The future API server componentconfig code should be compatible with the proposed structure\n\n### Non-goals\n\n* Graduate the API versions\n    * For v1.12, we’re working incrementally and will keep the API versions of the existing ComponentConfigs.\n* Do major refactoring of the ComponentConfigs. This PR is about code moves, not about re-defining the structure. We will do the latter in follow-ups.\n* Change the components to support reading a config file, do flag precedence correctly or add e2e testing\n    * Further, the \"load-versioned-config-from-flag\" feature in this proposal *should not* be confused with the\n      [Dynamic Kubelet Configuration](https://kubernetes.io/docs/tasks/administer-cluster/reconfigure-kubelet/) feature. There is nothing in this proposal advocating for that\n      a component should support a similar feature. This is all about the making the one-off “read bytes from a source and unmarshal into internal config state” possible for\n      both the internal components and external consumers of these APIs\n    * This is work to be done after this proposal is implemented (for every component but the kubelet which has this implemented already), and might or might not require\n      further, more component-specific proposals/KEPs\n* Create a net-new ComponentConfiguration struct for the API server\n* Publish the internal types to the new `k8s.io/{component}` repo\n* Support ComponentConfiguration for the cloud-controller-manager, as it’s a stop-gap for the cloud providers to move out of tree. This effort is in progress.\n    * When the currently in-tree cloud providers have move out of tree, e.g. to `k8s.io/cloud-provider-gcp`, they should create their own external and internal types and\n      make the command support loading configuration files.\n    * The new repo can reuse the generic types from the to-be-created, `k8s.io/controller-manager` repo eventually.\n    * Meanwhile, the cloud-controller-manager will reference the parts it needs from the main repo, and live privately in `cmd/cloud-controller-manager`\n* Expose defaulting functions for the external ComponentConfig types in `k8s.io/{component}/config` packages.\n    * Defaulting functions will still live local to the component in e.g. `k8s.io/kubernetes/pkg/{component}/apis/config/{version}` and be\n      registered in the default scheme, but won't be publicly exposed in the `k8s.io/{component}` repo.\n    * The only defaulting functions that are published to non-core repos are for the shared config types, in other words in\n      `k8s.io/{apimachinery,apiserver,controller-manager}/pkg/apis/config/{version}`, but **they are not registered in the scheme by default**\n      (with the normal `SetDefault_Foo` method and the `addDefaultingFunc(scheme *runtime.Scheme) { return RegisterDefaults(scheme) }` function).\n      Instead, there will be `RecommendedDefaultFoo` methods exposed, which the consumer of the shared types may or may not manually run in\n      `SetDefaults_Bar` functions (where `Bar` wraps `Foo` as a field).\n\n### Related proposals and further references\n\n* [Original Google Docs version of this KEP](https://docs.google.com/document/d/1-u2y03ufX7FzBDWv9dVI_HyiIQZL8iz3o3vabeOpP5Y/edit)\n* [Kubernetes Component Configuration](https://docs.google.com/document/d/1arP4T9Qkp2SovlJZ_y790sBeiWXDO6SG10pZ_UUU-Lc/edit) by [@mikedanese](https://github.com/mikedanese)\n* [Versioned Component Configuration Files](https://docs.google.com/document/d/1FdaEJUEh091qf5B98HM6_8MS764iXrxxigNIdwHYW9c/edit#) by [@mtaufen](https://github.com/mtaufen)\n* [Creating a ComponentConfig struct for the API server](https://docs.google.com/document/d/1fcStTcdS2Foo6dVdI787Dilr0snNbqzXcsYdF74JIrg/edit) by [@luxas](https://github.com/luxas) \u0026 [@sttts](https://github.com/sttts)\n* Related tracking issues in kubernetes/kubernetes:\n    * [Move `KubeControllerManagerConfiguration` to `pkg/controller/apis/`](https://github.com/kubernetes/kubernetes/issues/57618)\n    * [kube-proxy config should move out of ComponentConfig apigroup](https://github.com/kubernetes/kubernetes/issues/53577)\n    * [Move ClientConnectionConfiguration struct to its own api group](https://github.com/kubernetes/kubernetes/issues/54318)\n    * [Move `CloudControllerManagerConfiguration` to `cmd/cloud-controller-manager/apis`](https://github.com/kubernetes/kubernetes/issues/65458)\n\n## Proposal\n\n* for component in [kubelet kubeproxy kubecontrollermanager kubeapiserver kubescheduler]\n    * API group name: `{component}.config.k8s.io`\n    * Kind name: `{Component}Configuration`\n    * Code location:\n        * External types: `k8s.io/{component}/config/{version}/types.go`\n            * Like `k8s.io/api`\n        * Internal types: `k8s.io/kubernetes/pkg/{component}/apis/config`\n            * Alternatives, if applicable\n                * `k8s.io/{component}/pkg/apis/config` (preferred, in the future)\n                * `k8s.io/kubernetes/cmd/{component}/app/apis/config`\n            * If dependencies allow it, we can move them to `k8s.io/{component}/pkg/apis/config/types.go`. Not having the external types there is intentional because the `pkg/` package tree is considered as \"on-your-own-risks / no code compatibility guarantees\", while `config/` is considered as a code API.\n        * Internal scheme package: `k8s.io/kubernetes/pkg/{component}/apis/config/scheme/scheme.go`\n            * The scheme package should expose `Scheme *runtime.Scheme`, `Codecs *serializer.CodecFactory`, and `AddToScheme(*runtime.Scheme)`, and have an `init()` method that runs `AddToScheme(Scheme)`\n    * For the move to a staging repo to be possible, the external API package must not depend on the core repo.\n        * Hence, all non-staging repo dependencies need to be removed/resolved before the package move.\n    * Conversions from the external type to the internal type will be kept in `{internal_api_path}/{external_version}`, like for `k8s.io/api`\n        * Defaulting code will be kept in this package, besides the conversion functions.\n        * The defaulting code here is specific for the usage of the component, and internal by design. If there are defaulting functions we\n          feel would be generally useful, they might be exposed in `k8s.io/{component}/config/{version}/defaults.go` as `RecommendedDefaultFoo`\n          functions that can be used by various consumers optionally.\n    * Add at least some kind of minimum validation coverage for the types (e.g. ranges for integer values, URL scheme/hostname/port parsing,\n      DNS name/label/domain validation) in the `{internal_api_path}/validation` package, targeting the internal API version. The consequence of\n      these validations targeting the internal type is that they can't be exposed in the `k8s.io/{component}` repo, but publishing more\n      functionality like that is out of scope for this KEP and left as a future task. \n* Create a \"shared types\"-package with structs generic to all or many componentconfig API groups, in the `k8s.io/apimachinery`, `k8s.io/apiserver` and `k8s.io/controller-manager` repos, depending on the struct.\n    * Location: `k8s.io/{apimachinery,apiserver,controller-manager}/pkg/apis/config/{,v1alpha1}`\n    * These aren’t \"real\" API groups, but they have both internal and external versions\n    * Conversions and internal types are published to the staging repo.\n    * Defaulting functions are of the `RecommendedDefaultFoo` format and opt-ins for consumers. No defaulting functions are registered in the scheme.\n* Remove the monolithic `componentconfig/v1alpha1` API group (`pkg/apis/componentconfig`)\n* Enable the staging bot to create the Github repos\n* Add API roundtrip (fuzzing), defaulting, conversion, JSON tag consistency and validation tests.\n\n### Migration strategy per component or k8s.io repo\n\n#### k8s.io/apimachinery changes\n\n* **Not a \"real\" API group, instead shared packages only with both external and internal types.**\n* **External Package with defaulting (where absolutely necessary) \u0026 conversions**: `k8s.io/apimachinery/pkg/apis/config/v1alpha1/types.go`\n* **Internal Package**: `k8s.io/apimachinery/pkg/apis/config/types.go`\n* Structs to be hosted initially:\n    * ClientConnectionConfiguration\n* Assignee: @hanxiaoshuai\n\n#### k8s.io/apiserver changes\n\n* **Not a \"real\" API group, instead shared packages only with both external and internal types.**\n* **External Package with defaulting (where absolutely necessary) \u0026 conversions**: `k8s.io/apiserver/pkg/apis/config/v1alpha1/types.go`\n* **Internal Package**: `k8s.io/apiserver/pkg/apis/config/types.go`\n* Structs to be hosted initially:\n    * `LeaderElectionConfiguration`\n    * `DebuggingConfiguration`\n    * later to be created: SecureServingConfiguration, AuthenticationConfiguration, AuthorizationConfiguration, etc.\n* Assignee: @hanxiaoshuai\n\n#### kubelet changes\n\n* **GroupVersionKind:** `kubelet.config.k8s.io/v1beta.KubeletConfiguration`\n* **External Package:** `k8s.io/kubelet/config/v1beta1/types.go`\n* **Internal Package:** `k8s.io/kubernetes/pkg/kubelet/apis/config/types.go`\n* **Internal Scheme:** `k8s.io/kubernetes/pkg/kubelet/apis/config/scheme/scheme.go`\n* **Conversions \u0026 defaulting (where absolutely necessary) Package:** `k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1`\n* **Future Internal Package:** `k8s.io/kubelet/pkg/apis/config/types.go`\n* Assignee: @mtaufen\n\n#### kube-proxy changes\n\n* **GroupVersionKind**: `kubeproxy.config.k8s.io/v1alpha1.KubeProxyConfiguration`\n* **External Package**: `k8s.io/kube-proxy/config/v1alpha1/types.go`\n* **Internal Package**: `k8s.io/kubernetes/pkg/proxy/apis/config/types.go`\n* **Internal Scheme**: `k8s.io/kubernetes/pkg/proxy/apis/config/scheme/scheme.go`\n* **Conversions \u0026 defaulting (where absolutely necessary) Package:** `k8s.io/kubernetes/pkg/proxy/apis/config/v1alpha1`\n* **Future Internal Package:** `k8s.io/kube-proxy/pkg/apis/config/types.go`\n* Start referencing `ClientConnectionConfiguration` from the generic ComponentConfig packages\n* Assignee: @m1093782566\n\n#### kube-scheduler changes\n\n* **GroupVersionKind**: `kubescheduler.config.k8s.io/v1alpha1.KubeSchedulerConfiguration`\n* **External Package**: `k8s.io/kube-scheduler/config/v1alpha1/types.go`\n* **Internal Package**: `k8s.io/kubernetes/pkg/scheduler/apis/config/types.go`\n* **Internal Scheme**: `k8s.io/kubernetes/pkg/scheduler/apis/config/scheme/scheme.go`\n* **Conversions \u0026 defaulting (where absolutely necessary) Package:** `k8s.io/kubernetes/pkg/scheduler/apis/config/v1alpha1`\n* **Future Internal Package:** `k8s.io/kube-scheduler/pkg/apis/config/types.go`\n* Start referencing `ClientConnectionConfiguration` \u0026 `LeaderElectionConfiguration` from the generic ComponentConfig packages\n* Assignee: @dixudx\n\n#### k8s.io/controller-manager changes\n\n* **Not a \"real\" API group, instead shared packages only with both external and internal types.**\n* **External Package with defaulting (where absolutely necessary) \u0026 conversions**: `k8s.io/controller-manager/pkg/apis/config/v1alpha1/types.go`\n* **Internal Package**: `k8s.io/controller-manager/pkg/apis/config/types.go`\n* Will host structs:\n    * `GenericComponentConfiguration` (which will be renamed to `GenericControllerManagerConfiguration`)\n* Assignee: @stewart-yu\n\n#### kube-controller-manager changes\n\n* **GroupVersionKind**: `kubecontrollermanager.config.k8s.io/v1alpha1.KubeControllerManagerConfiguration`\n* **External Package**: `k8s.io/kube-controller-manager/config/v1alpha1/types.go`\n* **Internal Package**: `k8s.io/kubernetes/pkg/controller/apis/config/types.go`\n* **Internal Scheme**: `k8s.io/kubernetes/pkg/controller/apis/config/scheme/scheme.go`\n* **Conversions \u0026 defaulting (where absolutely necessary) Package:** `k8s.io/kubernetes/pkg/controller/apis/config/v1alpha1`\n* **Future Internal Package:** `k8s.io/kube-controller-manager/pkg/apis/config/types.go`\n* Start referencing `ClientConnectionConfiguration` \u0026 `LeaderElectionConfiguration` from the generic ComponentConfig packages\n* Assignee: @stewart-yu\n\n#### cloud-controller-manager changes\n\n* **Not a \"real\" API group, instead only internal types in `cmd/`.**\n* **Internal Package:** `cmd/cloud-controller-manager/app/apis/config/types.go`\n* We do not plan to publish any external types for this in a staging repo.\n* The internal cloud-controller-manager ComponentConfiguration types will reference both `k8s.io/controller-manager/pkg/apis/config`\n  and `k8s.io/kubernetes/pkg/controller/apis/config/`\n* Assignee: @stewart-yu\n\n#### kube-apiserver changes\n\n* **Doesn’t have a ComponentConfig struct at the moment, so there is nothing to move around.**\n* Eventually, we want to create this ComponentConfig struct, but exactly how to do that is out of scope for this specific proposal.\n* See [Creating a ComponentConfig struct for the API server](https://docs.google.com/document/d/1fcStTcdS2Foo6dVdI787Dilr0snNbqzXcsYdF74JIrg/edit) for a proposal on how to refactor the API server code to be able to expose the final ComponentConfig structure.\n\n### Vendoring\n\n#### Vgo\n\nVgo – as the future standard vendoring mechanism in Golang – supports [vgo modules](https://research.swtch.com/vgo-module) using a `k8s.io/{component}/config/go.mod` file. Tags of the shape `config/vX.Y` on `k8s.io/{component}` will define a version of the component config of that component. Such a tagged module can be imported into a 3rd-party program without inheriting dependencies outside of the `k8s.io/{component}/config` package.\n\nThe `k8s.io/{component}/config/go.mod` file will look like this:\n\n```\nmodule \"k8s.io/{component}/config\"\nrequire (\n\t\"k8s.io/apimachinery\" v1.12.0\n)\n```\n\nThe exact vgo semver versioning scheme we will use is out of scope of this document. We will be able to version the config package independently from the main package `k8s.io/{component}` if we want to, e.g. to implement correct semver semantics.\n\nOther 3rd-party code can import the config module as usual. Vgo does not add the dependencies from code outside of `k8s.io/{component}/config` (actually, vgo creates a separate `vgo.sum` for the config package with the transitive dependencies).\n\nCompare http://github.com/sttts/kubeadm for a test project using latest vgo.\n\n#### Dep\n\nDep supports the import of sub-packages without inheriting dependencies from outside of the sub-package.\n\n### Timeframe and Implementation Order\n\nObjective: Done for v1.12\n\nImplementation order:\n* Start with copying over the necessary structs to the `k8s.io/apiserver` and `k8s.io/apimachinery ` shared config packages, with external and internal API versions. The defaulting pattern for these types are of the `RecommendedDefaultFoo` form, in other words defaulting is not part of the scheme.\n* Remove as many unnecessary references to `pkg/apis/componentconfig` from the rest of the core repo as possible\n* Make the types in `pkg/apis/componentconfig` reuse the newly-created types in `k8s.io/apiserver` and `k8s.io/apimachinery `.\n* Start with the scheduler as the first component to be moved out.\n    * One PR for moving `KubeSchedulerConfiguration` to `staging/src/k8s.io/kube-scheduler/config/v1alpha1/types.go`, and the internal type to `pkg/scheduler/apis/config/types.go`.\n        * Set up the conversion for the external type by creating the package `pkg/scheduler/apis/config/v1alpha1`, without `types.go`, like how `k8s.io/api` is set up.\n        * This should be a pure code move.\n* Set up staging publishing bot (async, non-critical)\n* The kubelet, kube-proxy and kube-controller-manager types follow, each one independently.\n\n### OWNERS files for new packages and repos\n\n* Approvers:\n    * @kubernetes/api-approvers\n    * @sttts\n    * @luxas\n    * @mtaufen\n* Reviewers:\n    * @kubernetes/api-reviewers\n    * @sttts\n    * @luxas\n    * @mtaufen\n    * @dixudx\n    * @stewart-yu\n"
+  },
+  {
+    "id": "20322619a484e5d26cf9b0658d6c772e",
+    "title": "Create a `k8s.io/component-base` repo",
+    "authors": ["@luxas", "@sttts"],
+    "owningSig": "sig-cluster-lifecycle",
+    "participatingSigs": ["sig-api-machinery", "sig-cloud-provider"],
+    "reviewers": [
+      "@thockin",
+      "@jbeda",
+      "@bgrant0607",
+      "@smarterclayton",
+      "@liggitt",
+      "@lavalamp",
+      "@andrewsykim",
+      "@cblecker"
+    ],
+    "approvers": ["@thockin", "@jbeda", "@bgrant0607", "@smarterclayton"],
+    "editor": "@luxas",
+    "creationDate": "2018-11-27",
+    "lastUpdated": "2018-12-10",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Create a `k8s.io/component-base` repo\n\n**How we can consolidate the look and feel of core and non-core components with regards to ComponentConfiguration, flag handling, and common functionality with a new repository**\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Abstract](#abstract)\n  - [History and Motivation](#history-and-motivation)\n  - [\u0026quot;Component\u0026quot; definition](#component-definition)\n  - [Goals](#goals)\n  - [Success metrics](#success-metrics)\n  - [Non-goals](#non-goals)\n  - [Related proposals / references](#related-proposals--references)\n- [Proposal](#proposal)\n  - [Part 1: ComponentConfig](#part-1-componentconfig)\n    - [Standardized encoding/decoding](#standardized-encodingdecoding)\n    - [Testing helper methods](#testing-helper-methods)\n    - [Generate OpenAPI specifications](#generate-openapi-specifications)\n  - [Part 2: Command building / flag parsing for long-running daemons](#part-2-command-building--flag-parsing-for-long-running-daemons)\n    - [Wrapper around cobra.Command](#wrapper-around-cobracommand)\n    - [Flag precedence over config file](#flag-precedence-over-config-file)\n    - [Standardized logging](#standardized-logging)\n  - [Part 3: HTTPS serving](#part-3-https-serving)\n    - [Common endpoints](#common-endpoints)\n    - [Standardized authentication / authorization](#standardized-authentication--authorization)\n  - [Part 4: Sample implementation in k8s.io/sample-component](#part-4-sample-implementation-in-k8siosample-component)\n  - [Code structure](#code-structure)\n  - [Timeframe and Implementation Order](#timeframe-and-implementation-order)\n  - [OWNERS file for new packages](#owners-file-for-new-packages)\n\u003c!-- /toc --\u003e\n\n## Abstract\n\nThe proposal is about refactoring the Kubernetes core package structure in a way that all core component can share common code around\n\n- ComponentConfig implementation\n- flag and command handling\n- HTTPS serving\n- delegated authn/z\n- logging.\n\nToday this code is spread over the `k8s.io/kubernetes` repository, staging repository or pieces of code are in locations they don't belong to (example: `k8s.io/apiserver/pkg/util/logs` is \nthe for general logging, totally independent of API servers). We miss a repository far enough in the dependency hierarchy for code that is or should be common among core Kubernetes \ncomponent (neither `k8s.io/apiserver`, `k8s.io/apimachinery` or `k8s.io/client-go` are right for that).\n\nThis toolkit with this shared set of code can then be consumed by all the core Kubernetes components, higher-level frameworks like `kubebuilder` and `server-sdk` (which are more targeted \nfor a specific type of consumer), as well as any other ecosystem component that want to follow these patterns by vendoring this code as-is.\n\nTo implement this KEP in a timely manner and to start building a good foundation for Kubernetes component, we propose to create a Working Group, **WG Component Standard** to facilitate \nthis effort.\n\n### History and Motivation\n\nBy this time in the Kubernetes development, we know pretty well how we want a Kubernetes component to work, function, and look. But achieving this requires a fair amount of more or less\nadvanced code. As we scale the ecosystem, and evolve Kubernetes to work more as a kernel, it's increasingly important to make writing extensions and custom Kubernetes-aware components\nrelatively easy. As it stands today, this is anything but straightforward. In fact, even the in-core components diverge in terms of configurability (Can it be declaratively configured? Do \nflag names follow a consistent pattern? Are configuration sources consistently merged?), common functionality (Does it support the common \"/version,\" \"/healthz,\" \"/configz,\" \"/pprof,\" and \n\"/metrics\" endpoints? Does it utilize Kubernetes' authentication/authorization mechanisms? Does it write logs in a consistent manner? Does it handle signals as others do?), and testability \n(Do the internal configuration structs set up correctly to conform with the Kubernetes API machinery, and have roundtrip, defaulting, validation unit tests in place? Does it merge flags \nand the config file correctly? Is the logging mechanism set up in a testable manner? Can it be verified that the HTTP server has the standard endpoints registered and working? Can it be \nverified that authentication and authorization is set up correctly?).\n\n![component architecture](component-arch.png)\n\nThis document proposes to create a new Kubernetes staging repository with minimal dependencies (_k8s.io/apimachinery_, _k8s.io/client-go_, and _k8s.io/api_) and good documentation on how \nto write a Kubernetes-aware component that follows best practices. The code and best practices in this repo would be used by all the core components as well. Unifying the core components \nwould be great progress in terms of the internal code structure, capabilities, and test coverage. Most significantly, this would lead to an adoption of ComponentConfig for all internal \ncomponents as both a \"side effect\" and a desired outcome, which is long time overdue.\n\nThe current inconsistency is a headache for many Kubernetes developers, and confusing for end users. Implementing this proposal will lead to better code quality, higher test coverage in\nthese specific areas of the code, and better reusability possibilities as we grow the ecosystem (e.g. breaking out the _cloud provider_ code, building Cluster API controllers, etc.). \nThis work consists of three major pillars, and we hope to complete at least the ComponentConfig part of it—if not (ideally) all three pieces of work—in v1.14.\n\n### \"Component\" definition\n\nIn this case, when talking about a _component_, we mean: \"a CLI tool or a long-running server process that consumes configuration from a versioned configuration file and optionally \noverriding flags\". The component's implementation of ComponentConfig and command \u0026amp; flag setup is well unit-tested. The component is to some extent Kubernetes-aware. The\ncomponent follows Kubernetes' conventions for config serialization and merging, logging, and common HTTPS endpoints.\n\nWhen we say _core Kubernetes components_ we refer to: kube-apiserver, kube-controller-manager, kube-scheduler, kubelet, kube-proxy, and kubeadm.\n\n_To begin with_, this proposal will focus on **factoring out the code needed for the core Kubernetes components**. As we go, however, this set of packages will become\ngeneric enough to be usable by cloud provider and Cluster API controller extensions, as well as aggregated API servers.\n\n### Goals\n\n- Make it easy for a component to correctly adopt ComponentConfig (encoding/decoding/flag merging).\n- Avoid moving code into _k8s.io/apiserver_ which does not strictly belong to an etcd-based, API group-serving apiserver. Corollary: remove etcd dependency from components.\n- Factor out command- and flag-building code to a shared place.\n- Factor out common HTTPS endpoints describing a component's status.\n- Make the core Kubernetes components utilize these new packages.\n- Have good documentation about how to build a component with a similar look and feel as core Kubernetes components.\n- Increase test coverage for the configuration, command building, and HTTPS server areas of the component code.\n- Break out OpenAPI definitions and violations for ComponentConfigs from the monorepo to a dedicated place per-component.\n- Create a new Working Group, `WG Component Standard`, to facilitate and implement this and future related KEPs\n- Integrate well with related higher-level framework projects,  e.g. `kubebuilder` or `controller-runtime`\n  - This repo will host a set of packages needed by the core components in a generic manner, like a **generic toolkit**.\n  - More higher-level and scoped projects like `kubebuilder` will focus specifically on one target consumer, vendor this code\n    and wrap it in a way that makes sense for that specific consumer. That is what we like to refer to here as a **framework**.\n\n### Success metrics\n\n- All core Kubernetes components (kube-apiserver, kube-controller-manager, kube-scheduler, kubelet, kube-proxy, kubeadm) are using these shared packages in a consistent manner.\n- Cloud providers can be moved out of core without having to depend on the core repository.\n  - Related issue: [https://github.com/kubernetes/kubernetes/issues/69585](https://github.com/kubernetes/kubernetes/issues/69585)\n- It's easier for _kubeadm_ to move out of the core repo when these component-related packages are in a \"public\" staging repository.\n- `k8s.io/apiserver` doesn't have any code that isn't strictly related to serving API groups.\n\n### Non-goals\n\n- Graduate any ComponentConfig API versions (in this proposal).\n- Make this library toolkit a \"generic\" cloud-native component builder. Such a framework, if ever created, could instead consume these packages.\n  - We'll collaborate with higher-level projects like `kubebuilder` and `controller-runtime` though, and let them implement some of the higher-level code out of scope for this repo.\n- Fixing _all the problems_ in the core components, and expanding this beyond what's really necessary.\n  - Instead we'll work incrementally, and start with breaking out some basic stuff we _know_ every component must handle (e.g. configuration and flag parsing)\n- Specifying (in this proposal) _exactly how a Kubernetes component should function_. That is to be followed up later by the new Working Group.\n\n### Related proposals / references\n\n- [Kubernetes Component Configuration](https://docs.google.com/document/d/1arP4T9Qkp2SovlJZ_y790sBeiWXDO6SG10pZ_UUU-Lc/edit) by [@mikedanese](https://github.com/mikedanese)\n- [Versioned Component Configuration Files](https://docs.google.com/document/d/1FdaEJUEh091qf5B98HM6_8MS764iXrxxigNIdwHYW9c/edit#) by [@mtaufen](https://github.com/mtaufen)\n- [Moving ComponentConfig API types to staging repos](https://github.com/kubernetes/community/blob/master/keps/sig-cluster-lifecycle/0014-20180707-componentconfig-api-types-to-staging.md) by [@luxas](https://github.com/luxas) and [@sttts](https://github.com/sttts)\n - [Graduating KubeletFlags subfields to KubeletConfiguration](https://docs.google.com/document/d/18-MsChpTkrMGCSqAQN9QGgWuuFoK90SznBbwVkfZryo/edit) by [@mtaufen](https://github.com/mtaufen)\n - [Making /configz better](https://docs.google.com/document/d/1kNVSdw7H9EqyvI2BYH4EqwVtcg9bi9ZCPpJs8aKZFmM/edit) by [@mtaufen](https://github.com/mtaufen)\n - [Platform-Specific Component Configuration (draft)](https://docs.google.com/document/d/1rfSq9PGn_b7ILvWXjkgU9R6h5C84ywbKUrtJWo0IrYw/edit) by [@mtaufen](https://github.com/mtaufen)\n\n## Proposal\n\nThis proposal contains three logical units of work. Each subsection is explained in more detail below.\n\n### Part 1: ComponentConfig\n\n#### Standardized encoding/decoding\n\n- Encoding/decoding helper methods that would be referenced in every scheme package\n  - To make it easy to encode/decode a type registered in a scheme, `Encode`/`Decode` helper funcs would be registered in the scheme package\n  - Depending on the what these helper methods start to look like, they will be put in either `k8s.io/apimachinery` or `k8s.io/component-base`\n- Warn or (if desired, error) on unknown fields by creating a new strict universal codec in `k8s.io/apimachinery`\n  - This makes it possible for the component to spot e.g. config typos and notify the user instead of silently ignoring invalid config.\n  - More high-level, this codec can be used for e.g. a `--validate-config` flag\n- Support both JSON and YAML everywhere\n  - With 100% coverage on supporting both. If the component implements its own code something might go wrong so e.g. only YAML is supported\n- Support multiple YAML documents if needed\n  - For instance, supporting to read a directory of JSON/YAML files, or reading one single file with multiple YAML files\n\n#### Testing helper methods\n\n- Conversion / roundtrip testing\n  - Using YAML files in `testdata/` as well as roundtrip fuzzing testing\n- API group testing\n  - External types must have JSON tags\n  - Internal types must not have any JSON tags\n  - Verify expected API version name\n  - Verify expected API Group name\n  - Verify that the expected ComponentConfig type exists\n- Defaulting testing using YAML files in `testdata/`\n- Validation testing using YAML files in `testdata/`\n\n#### Generate OpenAPI specifications\n\nProvide a common way to generate OpenAPI specifications local to the component, so that external consumers can access it, and the component can expose it via e.g. a CLI flag or HTTPS \nendpoint.\n\nThe API naming violations handling (right now done monolithically in the core k8s repo) will become local to the component's API group. In other words, if a Go `CapitalCase` field name \ndoesn't have a similar JSON `camelCase` name, today an OpenAPI exception will be stored in `k8s.io/kubernetes/api/api-rules/violations_exception.list`. This will be changed\nfor components though, so that they register their allowed exceptions in the `register_test.go` unit test file in the external API group directory.\n\n### Part 2: Command building / flag parsing for long-running daemons\n\nGetting this code centralized and standardized will make it way easier for a server/daemon/controller component to be implmented.\nA more detailed document on _exactly how_ a Kubernetes server component should implement this is subject to a new KEP that will be created\nlater by the WG.\n\nPlease note that CLI tools are **not** targeted with this shared code here, only long-running daemons that basically only register flags to that one command.\n\n#### Wrapper around cobra.Command\n\nSee the `cmd/kubelet` code for how much extra setup a Kubernetes component needs to do for building commands and flag sets. This code can be refactored into a generic wrapper around \n_cobra_ for use with Kubernetes.\n\nNote: As part of follow-up proposals from the new Working Group, we might also consider using only `pflag` for server/daemon components.\n\n#### Flag precedence over config file\n\nIf the component supports both ComponentConfiguration and flags, flags should override fields set in the ComponentConfiguration. This is not straightforward to implement in code, and only \nthe kubelet does this at the moment. Refactoring this code in a generic helper library in this new repository will make adoption of the feature easy and testable.\n\nThe details of flag versus ComponentConfig semantics are _to be decided later in a different proposal_. Meanwhile, this flag precedence feature will be **opt-in**, so the kubelet and \nkubeadm can directly adopt this code, until the details have been decided on for all components.\n\n#### Standardized logging\n\nUse the _k8s.io/klog_ package in a standardized way.\n\n### Part 3: HTTPS serving\n\nMany Kubernetes controllers are clients to the API server and run as daemons. In order to expose information on how the component is doing (e.g. profiling, metrics, current configuration, \netc.), an HTTPS server is run.\n\n#### Common endpoints\n\nIn order to make it easy to expose this kind of information, a package is made in this new repo that hosts this common code. Initially targeted  \nendpoints are \"/version,\" \"/healthz,\" \"/configz,\" \"/pprof,\" and \"/metrics.\"\n\n#### Standardized authentication / authorization\n\nIn order to not expose this kind of information (e.g. metrics) to anyone that can talk to the component, it may utilize SubjectAccessReview requests to the API server, and hence delegate \nauthentication and authorization to the API server. It should be easy to add this functionality to your component.\n\n### Part 4: Sample implementation in k8s.io/sample-component\n\nProvides an example usage of the three main functions of the _k8s.io/component-base_ repo, implementing ComponentConfig, the CLI wrapper tooling and the common HTTPS endpoints with delegated \nauth.\n\n### Code structure\n\n- k8s.io/component-base\n  - config/\n    - Would hold internal, shared ComponentConfig types across core components\n    - {v1,v1beta1,v1alpha1}\n      - Would hold external, shared ComponentConfig types across core components\n    - serializer/\n      - Would hold common methods for encoding/decoding ComponentConfig\n    - testing/\n      - Would hold common testing code for use in unit tests local to the implementation of ComponentConfig.\n  - cli/\n    - Would hold common methods and types for building a k8s component command (building on top of github.com/spf13/{pflag,cobra})\n    - flags/\n      - Would hold flag implementations for custom types like `map[string]string`, tri-state string/bool flags, TLS cipher/version flags, etc.\n      - Code will be moved from `k8s.io/apiserver/pkg/util/{global,}flag`, `k8s.io/kubernetes/pkg/util/flag` and `k8s.io/kubernetes/pkg/version/verflag`.\n    - testing/\n      - Would hold common testing code for use in unit tests local to the implementation of the code\n    - logging/\n      - Would hold common code for using _k8s.io/klog_\n      - Code will be moved from `k8s.io/apiserver/pkg/util/logs`\n  - server/\n    - auth/\n      - Would hold code for implementing delegated authentication and authorization to Kubernetes\n      - authorizer/\n        - factory/\n          - Code will be moved from `k8s.io/apiserver/pkg/authorization/authorizationfactory/delegating.go`\n        - delegating/\n          - Code will be moved from `k8s.io/apiserver/plugin/pkg/authorizer/webhook`\n        - options/\n          - Code will be moved from `k8s.io/apiserver/pkg/server/options/authorization.go`\n      - authentication/\n        - factory/\n          - Code will be moved from `k8s.io/apiserver/pkg/authentication/authenticationfactory/delegating.go`\n        - delegating/\n          - Code will be moved from `k8s.io/apiserver/pkg/server/options/authentication.go`\n        - options/\n          - Code will be moved from `k8s.io/apiserver/pkg/server/options/authentication.go`\n    - configz/\n      - Would hold code for implementing a `/configz` endpoint in the component.\n      - Code will be moved from `k8s.io/kubernetes/pkg/util/configz`.\n      - Also consider moving the flag debugging endpoints in `k8s.io/apiserver/pkg/server/routes/flags.go` over here\n    - healthz/\n      - Would hold code for implementing a `/healthz` endpoint in the component\n      - Code will be moved from `k8s.io/apiserver/pkg/server/healthz`\n    - metrics/\n      - Would hold code for implementing a `/metrics` endpoint in the component\n      - Code will be moved from `k8s.io/kubernetes/pkg/util/metrics`,\n        `k8s.io/apiserver/pkg/server/routes/metrics.go`, and `k8s.io/apiserver/pkg/endpoints/metrics`\n    - openapi/\n      - Would hold code for implementing a `/openapi/v2` endpoint in the component\n      - Code will be moved from `k8s.io/apiserver/pkg/server/routes/openapi.go`\n    - pprof/\n      - Would hold code for implementing a `/pprof` endpoint in the component\n      - Code will be moved from `k8s.io/apiserver/pkg/server/routes/profiling.go`\n    - version/\n      - Would hold code for implementing a `/version` endpoint in the component\n      - Code will be moved from `k8s.io/apiserver/pkg/server/routes/version.go`\n    - signal/\n      - Would hold code to notify the command when SIGTERM or SIGINT happens, via a stop channel\n      - Code will be moved from `k8s.io/apiserver/pkg/server/signal*.go`\n\n### Timeframe and Implementation Order\n\n**Objective:** The ComponentConfig part done for v1.14\n\n**Stretch goal:** Get the CLI and HTTPS server parts done for v1.14.\n\n**Implementation priorities:**\n\nHere is a rough sketch of the priorities of the work for v1.14. Many of these items may be done in parallel.\nWe plan to \"go broad over deep\" here, in other words focus on implementing one step for each component before\nproceeding with the next task vs. doing everything for a component before proceeding to the next one.\n\n1. Create the `k8s.io/component-base` repo with the initial ComponentConfig shared code (e.g. packages earlier in `k8s.io/apiserver`)\n2. Move shared `v1alpha1` ComponentConfig types and references from `k8s.io/api{server,machinery}/pkg/apis/config` to `k8s.io/component-base/config`\n3. Set up good unit testing for all core ComponentConfig usage, by writing the `k8s.io/component-base/config/testing` package\n4. Move server-related util packages from `k8s.io/kubernetes/pkg/util/` to `k8s.io/component-base/server`. e.g. delegated authn/authz \"/configz\", \"/healthz\", and \"/metrics\" packages are suitable\n5. Move common flag parsing / cobra.Command setup code to `k8s.io/component-base/cli` from (mainly) the kubelet codebase.\n6. Start using the command- and server-related code in all core components.\n\nIn parallel to all the steps above, a _k8s.io/sample-component_ repo is built up with an example and documentation how to consume the _k8s.io/component-base_ code\n\n### OWNERS file for new packages\n\n- Approvers for the `k8s.io/component-base/config/{v1,v1beta1,v1alpha1}` packages\n  - @kubernetes/api-approvers\n- Approvers for `staging/src/k8s.io/{sample-component,component-base}`\n  - @sttts\n  - @luxas\n  - @jbeda\n  - @lavalamp\n- Approvers for moved subpackages:\n  - those who owned packages before code move\n- Reviewers for `staging/src/k8s.io/{sample-component,component-base}`:\n  - @sttts\n  - @luxas\n  - @dixudx\n  - @rosti\n  - @stewart-yu\n  - @dims\n- Reviewers for moved subpackages:\n  - those who owned packages before code move\n"
+  },
+  {
+    "id": "942bd054aa1e2e4956a8ddc4071ca66b",
+    "title": "Contributor Site",
+    "authors": ["@jbeda"],
+    "owningSig": "sig-contributor-experience",
+    "participatingSigs": ["sig-architecture", "sig-docs"],
+    "reviewers": ["@castrojo"],
+    "approvers": ["@parispittman"],
+    "editor": "TBD",
+    "creationDate": "2018-02-19",
+    "lastUpdated": "2018-03-07",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Contributor Site\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Drawbacks](#drawbacks)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nWe need a way to organize and publish information targeted at contributors.\nIn order to continue to scale the Kubernetes contributor community we need a convenient, scalable and findable way to publish information.\n\n## Motivation\n\nWhile the current kubernetes.io site is great for end users, it isn't often used by or aimed at project contributors.\nInstead, most contributors look at documentation in markdown files that are spread throughout a wide set of repos and orgs.\nIt is difficult for users to find this documentation.\n\nFurthermore, this documentation is often duplicated and out of date.\nThe fact that it isn't collected in one place and presented as a whole leads to fragmentation.\nOften times documentation will be duplicated because the authors themselves can't find the relevant docs.\n\nThis site will also serve as a starting point for those that are looking to contribute.\nThis site (and the contributor guide) can provide a soft introduction to the main processes and groups.\n\n\nFinally, some simple domain specific indexing could go a long way to make it easier to discover and cross link information.\nSpecifically, building a site that can take advantage of the KEP metadata will both make KEPs more discoverable and encourage those in the community to publish information in a way that *can* be discovered.\n\n### Goals\n\n* A contributor community facing portal to collect information for those actively working on upstream Kubernetes.\n* An easy to remember URL. (`contrib.kubernetes.io`? `contributors.kubernetes.io`? `c.kubernetes.io`?)\n* A streamlined process to update and share this information.\n  Ownership should be delegated using the existing OWNERS mechanisms.\n* A site that will be indexed well on Google to collect markdown files from the smattering of repos that we currently have.\n  This includes information that is currently in the [community repo](https://github.com/kubernetes/community).\n* Provide a place to launch and quickly evolve the contributor handbook.\n* Build some simple tools to enhance discoverability within the site.\n  This could include features such as automatically linking KEP and SIG names.\n* Over time, add an index of events, meetups, and other forums for those that are actively contributing to k8s.\n\n### Non-Goals\n\n* Actively migrate information from multiple orgs/repos.\n  This should be a place that people in the contributor community choose to use to communicate vs. being forced.\n* Create a super dynamic back end.  This is most likely served best with a static site.\n* Other extended community functions like a job board or a list of vendors.\n\n## Proposal\n\nWe will build a new static site out of the [community repo](https://github.com/kubernetes/community).\n\nThis site will be focused on communicating with and being a place to publish information for those that are looking to contribute to Kubernetes.\n\nWe will use Hugo and netlify to build and host the site, respectively. (Details TBD)\n\nThe main parts of the site that will be built out first:\n* A main landing page describing the purpose of the site.\n* A guide on how to contribute/update the site.\n* A list and index of KEPs\n* A place to start publishing and building the contributor guide.\n\n### Risks and Mitigations\n\nThe main risk here is abandonment and rot.\nIf the automation for updating the site breaks then someone will have to fix it.\nIf the people or the skillset doesn't exist to do so then the site will get out of sync with the source and create more confusion.\n\nTo mitigate this we will (a) ensure that SIG-contributor-experience is signed up to own this site moving forward and (b) keep it simple.\nBy relying on off the shelf tooling with many users (Hugo and Netlify) we can ensure that there are fewer custom processes and code to break.\nThe current generation scripts in the community repo haven't proven to be too much for us to handle.\n\n## Graduation Criteria\n\nThis effort will have succeeded if:\n\n* The contributor site becomes the de-facto way to publish information for the community.\n* People consistently refer to the contributor site when answering questions about \"how do I do X\" or \"what is the status of X\".\n* The amount of confusion over where to find information is reduced.\n* Others in the contributor community actively look to expand the information on the contributor site and move information from islands to this site.\n\n## Implementation History\n\n## Drawbacks\n\nThe biggest drawback is that this is yet another thing to keep running.\nCurrently the markdown files are workable but not super discoverable.\nHowever, they utilize the familiar mechanisms and do not require extra effort or understanding to publish.\n\nThe current mechanisms also scale across orgs and repos.\nThis is a strength as the information is close to the code but also a big disadvantage as it ends up being much less discoverable.\n\n## Alternatives\n\nOne alternative is to do nothing.\nHowever, the smattering of markdown through many repos is not scaling and is not discoverable via Google or for other members of the contributor community.\n\nThe main alternative here is to build something that is integrated into the user facing kubernetes.io site.\nThis is not preferred for a variety of reasons.\n\n* **Workflow.** Currently there is quite a bit of process for getting things merged into the main site.\n  That process involves approval from someone on SIG-Docs from an editorial point of view along with approval for technical accuracy.\n  The two stage approval slows down contributions and creates a much larger barrier than the current markdown based flow.\n  In addition, SIG-Docs is already stretched thin dealing with the (more important) user facing content that is their main charter.\n* **Quality standards.** The bar for the user facing site is higher than that of the contributor site.\n  Speed and openness of communication dominates for the contributor facing site.\n  Our bar here is the current pile of Markdown.\n* **Different tooling.** We may want to create specialized preprocessors as part of the contributor site build process.\n  This could include integrating our current expansion of sigs.yaml into Markdown files.\n  It may also include recognizing specific patterns (KEP-N) and creating automatic linkages.\n  Applying these to a part of a site or validating them across a larger site will slow creation of these tools.\n\nAn alternative to building directly into the website repo is to build in some other repo and do some sort of import into the main website repo.\nThere are serious downsides to this approach.\n\n* **No pre-commit visualization.** Netlifies capability to show a preview per PR won't work with a custom cross repo workflow.\n* **Higher latency for changes.** If the merges are batched and manually approved then there could be a significant time gap between when something is changed and when it is published.\n  This is a significant change from the current \"pile of markdown in github\" process.\n* **Opportunity for more complex build breaks.** If something is checked into a satellite repo it may pass all of the presubmit tests there but then fail presubmits on the parent repo.\n  This creates a situation where manual intervention is required.\n  Complicated pre-submit tests could be built for the satellite repo but those need to be maintained and debugged themselves.\n* **New tooling.** New tooling would need to be built that doesn't directly benefit the target audience.\n  This tooling will have to be documented and supported vs. using an off the shelf service like netlify.\n\n"
+  },
+  {
+    "id": "aca38a58bd2bef0fdd20813c807731cd",
+    "title": "A community forum for Kubernetes",
+    "authors": ["@castrojo"],
+    "owningSig": "sig-contributor-experience",
+    "participatingSigs": null,
+    "reviewers": ["@jberkus", "@joebeda", "@cblecker"],
+    "approvers": ["@parispittman", "@grodrigues3"],
+    "editor": "TBD",
+    "creationDate": "2018-04-03",
+    "lastUpdated": "2018-04-17",
+    "status": "implemented",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# A community forum for Kubernetes\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n  - [Implementation Details](#implementation-details)\n  - [Risks and Mitigations](#risks-and-mitigations)\n    - [References from other projects](#references-from-other-projects)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Drawbacks](#drawbacks)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nKubernetes is large enough that we should take a more active role in growing our community. We need a place to call our own that can encompass users, contributors, meetups, and other groups in the community. Is there a need for something between email and real time chat that can fulfill this? The primary purpose of this KEP is to determine whether we can provide a better community forum experience and perhaps improve our mailing list workflow. \n\nThe site would be forum.k8s.io, and would be linked to from the homepage and major properties. [See KEP005](https://github.com/kubernetes/community/blob/master/keps/sig-contributor-experience/0005-contributor-site.md) for related information on a contributor website. \n\n## Motivation\n\n- We're losing too much information in the Slack ether, and most of it does not show up in search engines. \n- Mailing lists remain mostly the domain of existing developers and require subscribing, whereas an open forum allows people to drive by and participate with minimal effort.\n  - There's an entire universe of users and developers that we could be reaching that didn't grow up on mailing lists and emacs. :D\n  - Specifically, hosting our lists on google groups has some issues:\n    - Automated filtering traps Zoom invites for SIG/WG leads\n    - Cannot use non-google accounts as first class citizens (Google Account required to create/manage group, join a group)\n    - Hard to search across multiple lists\n    - There's no way to see all the kubernetes lists in one view, we have to keep them indexed in sigs.yaml\n    - Filtering issues with the webui with countries that block Google \n    - Non-kubernetes branding\n- As part of a generic community portal, this gives people a place to go where they can discuss Kubernetes, and a sounding board for developers to make announcements of things happening around Kubernetes that can reach a wider audience.\n- We would be in charge of our own destiny, (aka. The Slack XMPP/IRC gateway removal-style concerns can be partially addressed)\n  - Software is 100% Open Source\n  - We'd have full access to our data, including the ability to export all of it.\n  - Kubernetes branded experience that would look professional and match the website and user docs look and feel, providing us a more consistent look across k8s properties. \n\n### Goals\n\n- Set up a prototype at discuss.k8s.io/discuss.kubernetes.io\n  - Determine if the mailing list feature is robust enough to replace our google groups\n  - References: [Mailing list roadmap](https://meta.discourse.org/t/moss-roadmap-mailing-lists/36432), [Discourse and email lists](https://meta.discourse.org/t/discourse-and-email-lists-like-google-groups/39915)\n- Heavy user engagement within 6 months.\n  - Clear usage growth in metrics\n    - Number of active users\n    - Number of threads \n    - Amount of traffic \n  - SIG Contributor Experience can analyze analytics regularly to determine growth and health. \n  - A feedback subforum would enable us to move quickly in addressing the needs of the community for the site. \n\n### Non-Goals\n\n- This is not a proposal to replace Slack, this is a proposal for a community forum.\n  - The motivation of having searchable information that is owned by the Kubernetes community comes from voiced concerns about having so much of Kubernetes depend on Slack. \n  - You are encouraged to propose a KEP for real-time communication if you would like to champion that, this KEP is not about Slack. \n- This does not replace Stack Overflow or kubernetes-users for user support. \n  - However inevitably users who prefer forums will undoubtedly use it for support.\n  - Strictly policing \"this should be posted here, that should be posted there\" won't work. \n  - I believe our community is large enough where we can have a support section, and there are enough people to make that self sustaining, we can also encourage cross posting from StackOverflow to integrate things better as both sites have good integration points.\n  - Over time as community interaction and knowledge base builds people will end up with a better experience than in #kubernetes-users on slack and will naturally gravitate there.\n  - Other large OSS communities have a presence on both StackOverflow and do support on their user forums already and it doesn't appear to be a big issue.\n- This will not replace kubernetes-devel or SIG mailing lists.\n  - They work, but we could experiment with mailing list integration. (See below)\n  - Let's concentrate on an end-user experience for now, and allow SIGs and working groups who want a more user-facing experience to opt-in if they wish. \n\n## Proposal\n\n### User Stories\n\n- A place for open ended discussion. For example \"What CI/CD tools is everyone using?\"\n  - This would be closed as offtopic on StackOverflow, but would be perfect for a forum.\n- Post announcements about kubernetes core that are important for end users\n  - An announcements subforum can be connected to Slack so that we have a single place for us to post announcements that get's propagated to other services.\n- Post announcements about related kubernetes projects\n  - Give the ecosystem of tools around k8s a place to go and build communities around all the tools people are building. \n  - \"Jill's neat K8s project on github\" is too small to have it's own official k8s presence, but it could be a post on a forum. \n- Events section for meetups and KubeCon/CloudNativeCon\n- Sub boards for meetup groups\n- Sub boards for non-english speaking community members\n- Developer section can include:\n  - Updated posting of community meeting notes\n  - We can inline the youtube videos as well \n  - Steering Committee announcements\n  - Link to important SIG announcements\n  - Any related user-facing announcements for our mentorship programs\n- Job board\n  - This might be difficult to do properly and keep classy, but leaving it here as a discussion point.\n- reddit.com/r/kubernetes has some great examples\n  - \"What are you working on this week?\" to spur activity\n  - \"So, what's the point of containerization?\" would be hard to scope on StackOverflow, but watercooly enough for a forum.\n  - [Top discussions](https://www.reddit.com/r/kubernetes/top/?t=month) over the last month. \n\n### Implementation Details\n\n- Software \n  - Discourse.org - https://www.discourse.org/features\n  - Free Software\n  - Vibrant community with lots of integrations with services we already use, like Slack and Github \n  - Rich API would allow us to build fun participatory integrations (User flair, contributor badges, etc.)\n  - Facilities for running polls and other plugins\n  - SSO with common login methods our community is already using.\n  - Moderation system is user-based on trust, so we would only need to choose 5 people as admins and then as people participate they build trust and get more admin responsibilities. \n  - Other developer and OSS communities such as Docker, Mozilla, Ubuntu, Twitter, Rust, and Atom are already effectively, software is mature and commonly used. \n  - Friendly upstream with known track record of working with other OSS projects.\n- Hosting \n  - We should host with a Discourse SaaS paid plan so we can concentrate on building the community and k8s itself.\n    - [Pricing information](https://payments.discourse.org/pricing)\n    - If other CNCF projects are interested in this we could help document best practices and do bulk pricing. \n  - Schedule regular dumps of our data to cloud provider buckets\n  - Exporting/Self Hosting is always available as an option \n  - Google Analytics integration would allow us to see what users are interested in and give us better insight on what is interesting to them. \n  - Data explorer so we can run ad hoc SQL queries and reports on the live data.\n  - Mailing list import would allow us to immediately have a searchable resource of our past activity. \n\n### Risks and Mitigations\n\n- One more thing to check everyday(tm)\n  - User fatigue with mailing lists, discourse, slack, stackoverflow, youtube channel, KubeCon/CloudNativeCon, your local meetup, etc.\n  - This is why I am proposing we investigate if we can replace the lists as well, two birds with one stone. \n- Lack of developer participation\n  - The mailing lists work, how suitable is Discourse to replace a mailing list these days? CNCF has tried Discourse in the past. See [@cra's post](https://twitter.com/cra/status/981548716405547008)\n  - [Discussion on the pros and cons of each](https://meta.discourse.org/t/discourse-vs-email-mailing-lists/54298)\n  - We have enough churn and new Working Groups that we could pilot a few, opt-in for SIGs that want to try it? \n- A community forum is asynchronous, whereas chat is realtime.\n  - This doesn't solve our Slack lock-in concerns, but can be a good first step in being more active in running our own community properties so that we can build out own resources. \n  - Ghost have [totally migrated to Discourse](https://twitter.com/johnonolan/status/980872508395188224?s=12) and shut down their Slack.\n    - We should keep an eye on this and see what data we can gleam from this. Engage with Ghost community folks to see what lessons they've learned.\n    - Not sure if getting rid of realtime chat entirely is a good idea either. \n- [GDPR Compliance](https://www.eugdpr.org/)\n  - Lots of data retention options in Discourse. \n  - We'd need to engage with upstream on their plans for this, we would want to avoid having to manage this ourselves. \n\n#### References from other projects\n\n- [Chef RFC](https://github.com/chef/chef-rfc/blob/master/rfc028-mailing-list-migration.md)\n  - [Blog post](https://coderanger.net/chef-mailing-list/) from a community member - good mailing list and community feedback here. \n- [Swift's Plan](https://lists.swift.org/pipermail/swift-evolution/Week-of-Mon-20170206/031657.html) - Long discussion, worth reading\n- [HTM Forum](https://discourse.numenta.org/t/guidelines-for-using-discourse-via-email/314)\n- [Julia](https://discourse.julialang.org/t/discourse-as-a-mailing-list/57) - It might be useful for us to investigate pregenerating the mail addresses?\n- [How's Discourse working out for Ghost](https://forum.ghost.org/t/hows-discourse-working-out-for-ghost/947) - We asked them for some direct feedback on their progress so far \n\n\n## Graduation Criteria\n\nThere will be a feedback subforum where users can directly give us feedback on what they'd like to see. Metrics and site usage should determine if this will be viable in the long term.\n\nAfter a _three month_ prototyping period SIG Contributor Experience will:\n\n- Determine if this is a better solution than what we have, and figure out where this would fit in the ecosystem\n  - There is a strong desire that this would replace an existing support venue, SIG Contributor Experience will weigh the options.  \n- If this solution is not better than what we have, and we don't want to support yet another tool we would shut the project down.\n- If we don't have enough information to draw a conclusion, we may decide to extend the evaluation period.\n- Site should have a moderation and administrative policies written down.\n\n\n## Implementation History\n\nMajor milestones in the life cycle of a KEP should be tracked in `Implementation History`.\nMajor milestones might include\n\n- the `Summary` and `Motivation` sections being merged signaling SIG acceptance\n- the `Proposal` section being merged signaling agreement on a proposed design\n- the date implementation started\n- the first Kubernetes release where an initial version of the KEP was available\n- the version of Kubernetes where the KEP graduated to general availability\n- when the KEP was retired or superseded\n\n## Drawbacks\n\n- Kubernetes has seen explosive growth without having a forum at all. \n"
+  },
+  {
+    "id": "60f4ec3c0e5189111f95af54cd57d359",
+    "title": "Leveraging Distributed Tracing to Understand Kubernetes Object Lifecycles",
+    "authors": ["@Monkeyanator"],
+    "owningSig": "sig-instrumentation",
+    "participatingSigs": [
+      "sig-architecture",
+      "sig-node",
+      "sig-api-machinery",
+      "sig-scalability",
+      "sig-cli"
+    ],
+    "reviewers": ["@Random-Liu", "@bogdandrutu"],
+    "approvers": ["@brancz", "@piosz"],
+    "editor": "@dashpole",
+    "creationDate": "2018-12-04",
+    "lastUpdated": "2019-11-07",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Leveraging Distributed Tracing to Understand Kubernetes Object Lifecycles\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Definitions](#definitions)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Architecture](#architecture)\n    - [Tracing API Requests](#tracing-api-requests)\n    - [Propagating Context Through Objects](#propagating-context-through-objects)\n    - [Controller Behavior](#controller-behavior)\n    - [End-User Behavior](#end-user-behavior)\n  - [In-tree changes](#in-tree-changes)\n    - [Vendor the Tracing Framework](#vendor-the-tracing-framework)\n    - [Trace Utility Package](#trace-utility-package)\n    - [Tracing Pod Lifecycle](#tracing-pod-lifecycle)\n  - [Out-of-tree changes](#out-of-tree-changes)\n    - [Tracing best-practices documentation](#tracing-best-practices-documentation)\n- [Graduation requirements](#graduation-requirements)\n- [Production Readiness Survey](#production-readiness-survey)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis Kubernetes Enhancement Proposal (KEP) introduces a model for adding distributed tracing to Kubernetes object lifecycles. The inclusion of this trace instrumentation will mark a significant step in making Kubernetes processes more observable, understandable, and debuggable.\n\n\n## Motivation\n\nDebugging latency issues in Kubernetes is an involved process. There are existing tools which can be used to isolate these issues in Kubernetes, but these methods fall short for various reasons. For instance:\n\n* **Logs**: are fragmented, and finding out which process was the bottleneck involves digging through troves of unstructured text. In addition, logs do not offer higher-level insight into overall system behavior without an extensive background on the process of interest. \n* **Events**: in Kubernetes are only kept for an hour by default, and don't integrate with visualization of analysis tools. To gain trace-like insights would require a large investment in custom tooling.\n* **Latency metrics**: can only supply limited metadata because of cardinality constraints.  They are useful for showing _that_ a process was slow, but don't provide insight into _why_ it was slow.\n* **Latency Logging**: is a \"poor man's\" version of tracing that only works within a single binary and outputs log messages.  See [github.com/kubernetes/utils/trace](https://github.com/kubernetes/utils/tree/master/trace).\n\nDistributed tracing provides a single window into latency information from across many components and plugins. Trace data is structured, and there are numerous established backends for visualizing and querying over it.\n\n### Definitions\n\n**Span**: The smallest unit of a trace.  It has a start and end time, and is attached to a single trace.\n**Trace**: A collection of Spans which represents a single process.\n**Trace Context**: A reference to a Trace that is designed to be propagated across component boundaries.\n\n### Goals\n\n* Make it possible to visualize the progress of objects across distinct Kubernetes components\n* Streamline the tedious processes involved in debugging latency issues in Kubernetes\n* Make it possible to identify high-level latency regressions, and attribute them back to offending processes\n\n\n### Non-Goals\n\n* Replace existing logging, metrics, or the events API\n* Trace operations from all Kubernetes resource types in a generic manner (i.e. without manual instrumentation)\n\n## Proposal\n\n### Architecture\n\n#### Tracing API Requests\n\nIn the traditional tracing model, a client sends a request to a server and receives a response back.  Even though Kubernetes \"controllers\" don't follow this model (more on that later), the kube-apierver and backing storage (e.g. etcd3) do.  To enable traces to be collected for API requests, the following must be true:\n\n1. The apiserver must propagate the http context of incoming requests through its function stack to the backing storage\n1. Kubernetes client libraries must allow passing a context with API requests\n\nTo actually add traces to API requests, owners of the kube-apiserver and backing storage may add Spans to incoming requests, and configure sampling as they see fit.\n\n#### Propagating Context Through Objects\n\nWhile API requests follow the traditional RPC client-server tracing model, kubernetes controllers don't.  Instead of controller actions being driven by incoming RPCs, their actions are driven by observations of desired and actual state.  This is the primary reason why the kubernetes community hasn't agreed on how to integrate tracing into kubernetes thus far.\n\nIn the traditional RPC client-server tracing model, a trace context is attached to a single incoming request, and is propagated with all requests the server makes to other servers required to fulfill the initial single request.  Conceptually, this proposal suggests treating a kubernetes cluster as a single RPC server.  The difference is that we attach context to objects, and propagate this context to objects modified as a result of the initial object modification.  For example, if a user creates a ReplicaSet, the kube-controller-manager will create many Pod objects as a result, and will propagate the context used to create the ReplicaSet to Pod objects as well.  This ensures that all actions taken by kubernetes controllers as a result of the initial user action are linked by the same context.\n\nFor the alpha phase, we choose to propagate this span context as an encoded string an object annotation called `trace.kubernetes.io/context`.  As noted in [Tracing API Requests](#tracing-api-requests) above, storing the trace context with the context is _in addition_ to attaching a context to http requests to the apiserver.  The reason for this is explained in the [Controller Behavior](#controller-behavior) section below.  In some scenarios, controllers will want to update the trace context from A -\u003e B, but want to associated that Update request with context A.\n\nThis means two trace contexts are sent in different forms with Create/Patch/Update requests to the apiserver.  A trace context is around 32 bytes (16 bytes for the trace ID, 8 bytes for the span ID, and some metadata). See the [w3c spec](https://w3c.github.io/trace-context/#tracestate-field) for details.\n\n\nThis annotation value is removed when an object's trace ends, to achieve the desired behavior from [section one](#trace-lifecycle).  For core kubernetes components, this must be done in the same request to the API Server as the status update which updates the object to its desired state.  This is a requirement to ensure tracing does not affect the scalability of kubernetes.  For other components, it is recommended, but not required to update the trace annotation in the same request.\n\nThis proposal chooses to use annotations to store the SpanContext associated with an object.  This mirrors how trace context propagation is done with golang context.Context and http headers, which are both key/value stores.\n\n#### Controller Behavior\n\nWhen reconciling an object `Foo` a Controller must:\n\n1. Send the trace context stored in `Foo` in the http request context for all API requests. See [Tracing API Requests](#tracing-api-requests)\n1. Store the trace context of `Foo` in object `Bar` when updating the Spec of `Bar`. See [Propagating Context Through Objects](#propagating-context-through-objects)\n1. Export a span around work that attempts to drive the actual state of an object towards its desired state\n1. Replace the trace context of `Foo` when updating `Foo`'s status to the desired state\n\nControllers must _only_ export Spans around work that it is correcting from an undesired state to its desired state.  To avoid exporting pointless spans, controllers must not export spans around reconciliation loops that do not perform actual work.  For example, the kubelet must not export a span around syncPod, which is a generic Reconcile function.  Instead, it should export spans around CreateContainer, or other functions that move the system towards its desired state. \n\nThis proposal is grounded on the principle that a trace context is attached to and propagated with end-user intent.  When the status of an object is updated to its desired state, the end-user's intent for that object has been fulfilled.  Controllers must \"end\" tracing for an object when it reaches its desired state.  To accomplish this, Controllers must update the trace context of an object when updating the status of an object from an undesired to a desired state.  For objects that report a status that can reach a desired state, this limits traces to just the actions taken by controllers in the fulfillment of the end-user's intent, and prevents traces from spanning an indefinite period of time.\n\nComponents should plumb the context through reconciliation functions, rather than storing and looking up trace contexts globally so that each attempt to reconcile desired and actual state uses the context associated with _that_ desired state through the entire attempt.  If multiple components are involved in reconciling a single object, one may act on the new trace context before the other, but each trace is still representative of the work done to reconcile to the corresponding desired state. Given this model, we guarantee that each trace contains the actions taken to reconcile toward a single desired state.\n\nHigh-level processes, such as starting a pod or restarting a failed container, could be interrupted before completion by an update to the desired state. While this leaves a \"partial\" trace for the first process, it is the most accurate representation of the work and timing of reconciling desired and actual state.\n\n#### End-User Behavior\n\nAdd a new `--trace` argument to `kubectl`, which generates a new trace context, sets the trace context to be sampled, attaches the context to all modified objects, and uses the context when sending requests to the API Server.  The option is disabled by default.  Note that by attaching a trace context to the initial object creation, this will cause all object modification done by controllers to propagate the context through to all changes made by the system that are driven by the initial user action.\n\nAdd `context.Context` arguments to k8s.io/client-go client functions.  This will allow users and components to associate API calls with the context of the involved object.  In some cases, such as object creation, we can automatically attach the SpanContext of the provided context to the created object, making propagation simpler.\n\nThis also enables kubernetes to be a composable part of a larger system. For example, if an end-user's service creates a pod as part of handing a request, it could do:\n```golang\nctx, span := trace.StartSpan(preexistingContext, “create-my-pod”)\ndefer span.End()\npod, err := c.CoreV1().Pod(myPod.Namespace).Create(ctx, myPod)\nif err != nil {\n    return err\n}\nwaitForPodToBeRunning(ctx, myPod)\nreturn nil\n```\n\nA previous iteration of this proposal suggested controllers should export a \"Root Span\" when ending a trace (described in [Controller Behavior](#controller-behavior) above).  However, that would limit a trace to being associated with a single object, since a \"Root Span\" defines the scope of the trace.  More generally, we shouldn't assume that the creation or update of a single object represents the entirety of end-user intent.  The user or system using kubernetes determines what the user intent is, not kubernetes controllers.\n\nTracing in a kubernetes cluster must be a composable component within a larger system, and allow external users or systems to define the \"Root Span\" that defines and bounds the scope of a trace.\n\n### In-tree changes\n\n#### Vendor the Tracing Framework\n\nThis KEP proposes the use of the [OpenTelemetry tracing framework](https://opentelemetry.io/) to create and export spans to configured backends.\n\nWhile in alpha, controllers should use the OpenTelemetry exporter, which exports traces to the [OpenTelemetry Collector](https://github.com/open-telemetry/opentelemetry-collector). The OpenTelemetry collector allows importing and configuring exporters for trace storage backends to be done out-of-tree in addition to other useful features.\n\nThis KEP suggests that we utilize the OpenTelemetry collector for the initial implementation to reduce the global changes required for alpha.  Alternative options include:\n\n1. Add configuration for exporters in-tree by vendoring in each \"supported\" exporter. These exporters are the only compatible backends for tracing in kubernetes.\n  a. This places the kubernetes community in the position of curating supported tracing backends\n  b. This eliminates the requirement to run to OpenTelemetry collector in order to use tracing\n2. Support *both* a curated set of in-tree exporters, and the collector exporter\n\nWhile this setup is suitable for an alpha stage, it will require further review from Sig-Instrumentation and Sig-Architecture for beta, as it introduces a dependency on the OT Collector.  It is also worth noting that OpenTelemetry still has many unresolved details on how to run the collector.\n\n#### Trace Utility Package\n\nThis package will be able to create spans from the span context embedded in the `trace.kubernetes.io/context` object annotation, in addition to embedding context from spans back into the annotation. This package will facilitate propagating traces through kubernetes objects.  The exported functions include:\n\n```golang\n// InitializeExporter initializes the trace exporting service with the provided service name.\n// Components should use this initializer to ensure common behavior.\nfunc InitializeExporter(service string)\n\n// StartSpanFromObject constructs a new Span using the context attached to the object as the parent SpanContext.  It mirrors trace.StartSpan, but for kubernetes objects.\nfunc StartSpanFromObject(ctx context.Context, obj meta.Object, spanName string) (context.Context, *trace.Span, error)\n\n// EncodeContextIntoObject encodes the SpanContext contained in the context into the object\nfunc EncodeContextIntoObject(ctx context.Context, obj meta.Object)\n\n// RemoveSpanContextFromObject removes the SpanContext attached to an object, if one exists\nfunc RemoveSpanContextFromObject(obj meta.Object) \n```\n\n#### Tracing Pod Lifecycle\n\nAs we move forward with this KEP, we will use the aforementioned trace utility package to trace pod-related operations across the scheduler and kubelet. In code, this corresponds to creating a span (i.e. `ctx, span := trace.StartSpan(ctx, \"Component.SampleSpan\")`) at the beginning of an operation, and ending the span afterwards (`span.End()`). All calls to tracing functions will be gated with the `ObjectLifecycleTracing` alpha feature-gate, and will be disabled by default.\n\nOpenTelemetry ships with plugins to transport trace context across gRPC and HTTP boundaries, which enables us to extend our tracing across the CRI and other internal boundaries.\n\nIn OpenTelemetry's Go implementation, span context is passed down through Go context. This will necessitate the threading of context across more of the Kubernetes codebase, which is a [desired outcome regardless](https://github.com/kubernetes/kubernetes/issues/815).\n\nWhile adding tracing to Pods is a good first step to demonstrate the viability of object lifecycle tracing in kubernetes, we expect component owners to add tracing to their components in an ad-hoc fashion.\n\n### Out-of-tree changes\n\n#### Tracing best-practices documentation\n\nThis KEP introduces a new form of instrumentation to Kubernetes, which necessitates the creation of guidelines for adding effective, standardized traces to Kubernetes components, [similar to what is found here for metrics](https://github.com/kubernetes/community/blob/master/contributors/devel/instrumentation.md).\n\nThis documentation will put forward standards for: \n\n* How to name spans, attributes, and annotations\n* What kind of processes should be wrapped in a span\n* When to link spans to other traces\n* What kind of data makes for useful attributes\n* How to propagate trace context as proposed above\n\nHaving these standards in place will ensure that our tracing instrumentation works well with all backends, and that reviewers have concrete criteria to cross-check PRs against. \n\n## Graduation requirements\n\nAlpha\n\n- [] Alpha-implementation as described above\n- [] E2e testing of traces\n- [] User-facing documentation\n- [] Tracing must not increase the number of requests to the APIServer\n\nBeta\n\n- [] Security Review, including threat model\n- [] Deployment review including whether the [OT Collector](https://github.com/open-telemetry/opentelemetry-collector) is a required component\n- [] Benchmark kubernetes components using tracing, and determine resource requirements and scaling for any additional required components (e.g. OT Collector).\n\nGA\n\n- [] Versioning for span naming and backwards-compatibility guarantees\n\n## Production Readiness Survey\n\n* Feature enablement and rollback\n  - How can this feature be enabled / disabled in a live cluster?  **Feature-gate: ComponentTracing.  All components that are instrumented with tracing must be restarted to enable/disable exporting spans from that component.  Initial components that will emit spans are: kube-apiserver, kube-scheduler, kube-controller-manager, kubelet.  Others may be added in later stages.**\n  - Can the feature be disabled once it has been enabled (i.e., can we roll\n    back the enablement)?  **Yes, the feature gate can be disabled in all relevant components**\n  - Will enabling / disabling the feature require downtime for the control\n    plane?  **Yes, control-plane components must be restarted with the feature-gate disabled.**\n  - Will enabling / disabling the feature require downtime or reprovisioning\n    of a node?  **No, it just requires restarting the kubelet with the feature-gate disabled**\n  - What happens if a cluster with this feature enabled is rolled back? What happens if it is subsequently upgraded again?  **No adverse effects in either case.**\n  - Are there tests for this?  **No.  The feature hasn't been developed yet.**\n* Scalability\n  - Will enabling / using the feature result in any new API calls? **No (there was a recent change in the KEP to that effect).**\n    Describe them with their impact keeping in mind the [supported limits][]\n    (e.g. 5000 nodes per cluster, 100 pods/s churn) focusing mostly on:\n     - components listing and/or watching resources they didn't before\n     - API calls that may be triggered by changes of some Kubernetes\n       resources (e.g. update object X based on changes of object Y)\n     - periodic API calls to reconcile state (e.g. periodic fetching state,\n       heartbeats, leader election, etc.)\n  - Will enabling / using the feature result in supporting new API types? **No**\n    How many objects of that type will be supported (and how that translates\n    to limitations for users)?\n  - Will enabling / using the feature result in increasing size or count\n    of the existing API objects?  **Yes.  It adds an annotation to \"traced\" objects.  The value is a trace context, which is ~32 bytes.  Traced objects will initially include pods, replicasets, and deployments, but may expand to include others over time.  Notably, this annotation should not be added to Events.**\n  - Will enabling / using the feature result in increasing time taken\n    by any operations covered by [existing SLIs/SLOs][] (e.g. by adding\n    additional work, introducing new steps in between, etc.)? **No**\n    Please describe the details if so.\n  - Will enabling / using the feature result in non-negligible increase\n    of resource usage (CPU, RAM, disk IO, ...) in any components?\n    Things to keep in mind include: additional in-memory state, additional\n    non-trivial computations, excessive access to disks (including increased\n    log volume), significant amount of data sent and/or received over\n    network, etc. Think through this in both small and large cases, again\n    with respect to the [supported limits][].  **The tracing client library has an in-memory cache for outgoing spans.  I believe this is limited to 1Mb by default.  This overhead would apply to all controllers that export spans.  Note that this applies to the kubelet as well, since the kubelet is one of the initial components that will be instrumented.**\n* Rollout, Upgrade, and Rollback Planning\n* Dependencies\n  - Does this feature depend on any specific services running in the cluster\n    (e.g., a metrics service)? **Yes.  In the current version of the proposal, users must run the [OpenTelemetry Collector](https://github.com/open-telemetry/opentelemetry-collector) as a daemonset to configure which backend (e.g. jager, zipkin, etc.) they want telemetry sent to.**\n  - How does this feature respond to complete failures of the services on\n    which it depends?  **Traces will stop being exported, and components will store spans in memory until the buffer is full.  After the buffer fills up, spans will be dropped.**\n  - How does this feature respond to degraded performance or high error rates\n    from services on which it depends? **If the bi-directional grpc streaming connection to the collector cannot be established or is broken, the controller retries the connection every 5 minutes (by default).**\n* Monitoring requirements\n  - How can an operator determine if the feature is in use by workloads?  **The operator can check for the presence of the trace context annotation.  Generally, operators are expected to have access to (and likely control over) the OpenTelemetry agent deployment and trace storage backend.**\n  - How can an operator determine if the feature is functioning properly?  **TODO: does the client library add metrics about trace exporting?**\n  - What are the service level indicators an operator can use to determine the\n    health of the service?  **Error rate of sending traces in controllers and OpenTelemetry collector.**\n  - What are reasonable service level objectives for the feature?  **Not entirely sure, but I would expect at least 99% of spans to be sent successfully, if not more.**\n* Troubleshooting\n  - What are the known failure modes?  **The controller is misconfigured, and cannot talk to the collector.  The collector is misconfigured, and can't send traces to the backend.**\n  - How can those be detected via metrics or logs?  Logs from the component or agent based on the failure mode.\n  - What are the mitigations for each of those failure modes?  **None.  You must correctly configure the collector for tracing to work.**\n  - What are the most useful log messages and what logging levels do they require? **All errors are useful, and are logged as errors (no logging levels required). Failure to initialize exporters (in both controller and collector), failures exporting metrics are the most useful.**\n  - What steps should be taken if SLOs are not being met to determine the\n    problem?  **Look at controller and collector logs.**\n\n## Implementation History\n\n* [Mutating admission webhook which injects trace context](https://github.com/Monkeyanator/mutating-trace-admission-controller)\n* [Instrumentation of Kubernetes components](https://github.com/Monkeyanator/kubernetes/pull/15)\n* [Instrumentation of Kubernetes components for 1/24/2019 community demo](https://github.com/kubernetes/kubernetes/compare/master...dashpole:tracing)\n"
+  },
+  {
+    "id": "eaf5ac386f2bf6901c408ed3239cf160",
+    "title": "Kubernetes Metrics Overhaul",
+    "authors": ["@brancz", "@ehashman"],
+    "owningSig": "sig-instrumentation",
+    "participatingSigs": null,
+    "reviewers": ["@piosz", "@DirectXMan12"],
+    "approvers": ["@piosz", "@DirectXMan12"],
+    "editor": "@DirectXMan12",
+    "creationDate": "2018-11-06",
+    "lastUpdated": "2019-03-21",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Kubernetes Metrics Overhaul\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [cAdvisor instrumentation changes](#cadvisor-instrumentation-changes)\n    - [Consistent labeling](#consistent-labeling)\n  - [Changing API latency histogram buckets](#changing-api-latency-histogram-buckets)\n  - [Kubelet metric changes](#kubelet-metric-changes)\n    - [Make metrics aggregatable](#make-metrics-aggregatable)\n    - [Export less metrics](#export-less-metrics)\n    - [Prevent apiserver's metrics from accidental registration](#prevent-apiservers-metrics-from-accidental-registration)\n    - [Prober metrics](#prober-metrics)\n  - [Kube-scheduler metric changes](#kube-scheduler-metric-changes)\n  - [Kube-proxy metric changes](#kube-proxy-metric-changes)\n    - [Change proxy metrics to conform metrics guidelines](#change-proxy-metrics-to-conform-metrics-guidelines)\n    - [Clean the deprecated metrics which introduced in v1.14](#clean-the-deprecated-metrics-which-introduced-in-v114)\n  - [Kube-apiserver metric changes](#kube-apiserver-metric-changes)\n    - [Apiserver and etcd metrics](#apiserver-and-etcd-metrics)\n    - [Fix admission metrics in true units](#fix-admission-metrics-in-true-units)\n    - [Remove the deprecated admission metrics](#remove-the-deprecated-admission-metrics)\n  - [Client-go metric changes](#client-go-metric-changes)\n    - [Workqueue metrics](#workqueue-metrics)\n  - [Convert latency/latencies in metrics name to duration](#convert-latencylatencies-in-metrics-name-to-duration)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Deprecation Plan](#deprecation-plan)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis Kubernetes Enhancement Proposal (KEP) outlines the changes planned in the scope of an overhaul of all metrics instrumented in the main kubernetes/kubernetes repository. This is a living document and as existing metrics, that are planned to change are added to the scope, they will be added to this document. As this initiative is going to affect all current users of Kubernetes metrics, this document will also be a source for migration documentation coming out of this effort.\n\nThis KEP is targeted to land in Kubernetes 1.14. The aim is to get all changes into one Kubernetes minor release, to have only a migration be necessary. We are preparing a number of changes, but intend to only start merging them once the 1.14 development window opens.\n\n## Motivation\n\nA number of metrics that Kubernetes is instrumented with do not follow the [official Kubernetes instrumentation guidelines](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/instrumentation.md). This is for a number of reasons, such as the metrics having been created before the instrumentation guidelines were put in place (around two years ago), and just missing it in code reviews. Beyond the Kubernetes instrumentation guidelines, there are several violations of the [Prometheus instrumentation best practices](https://prometheus.io/docs/practices/instrumentation/). In order to have consistently named and high quality metrics, this effort aims to make working with metrics exposed by Kubernetes consistent with the rest of the ecosystem. In fact even metrics exposed by Kubernetes are inconsistent in themselves, making joining of metrics difficult.\n\nKubernetes also makes extensive use of a global metrics registry to register metrics to be exposed. Aside from general shortcomings of global variables, Kubernetes is seeing actual effects of this, causing a number of components to use `sync.Once` or other mechanisms to ensure to not panic, when registering metrics. Instead a metrics registry should be passed to each component in order to explicitly register metrics instead of through `init` methods or other global, non-obvious executions. Within the scope of this KEP, we want to explore other ways, however, it is not blocking for its success, as the primary goal is to make the metrics exposed themselves more consistent and stable.\n\nWhile uncertain at this point, once cleaned up, this effort may put us a step closer to having stability guarantees for Kubernetes around metrics. Currently metrics are excluded from any kind of stability requirements.\n\n### Goals\n\n* Provide consistently named and high quality metrics in line with the rest of the Prometheus ecosystem.\n* Consistent labeling in order to allow straightforward joins of metrics.\n\n### Non-Goals\n\n* Add/remove metrics. The scope of this effort just concerns the existing metrics. As long as the same or higher value is presented, adding/removing may be in scope (this is handled on a case by case basis).\n* This effort does not concern logging or tracing instrumentation.\n\n## Proposal\n\n### cAdvisor instrumentation changes\n\n#### Consistent labeling\n\nChange the container metrics exposed through cAdvisor (which is compiled into the Kubelet) to [use consistent labeling according to the instrumentation guidelines](https://github.com/kubernetes/kubernetes/pull/69099). Concretely what that means is changing all the occurrences of the labels:\n`pod_name` to `pod`\n`container_name` to `container`\n\nAs Kubernetes currently rewrites meta labels of containers to “well-known” `pod_name`, and `container_name` labels, this code is [located in the Kubernetes source](https://github.com/kubernetes/kubernetes/blob/097f300a4d8dd8a16a993ef9cdab94c1ef1d36b7/pkg/kubelet/cadvisor/cadvisor_linux.go#L96-L98), so it does not concern the cAdvisor code base.\n\n### Changing API latency histogram buckets\n\nAPI server histogram latency buckets run from 125ms to 8s. This range does not accurately model most API server request latencies, which could run as low as 1ms for GETs or as high as 60s before hitting the API server global timeout.\n\nhttps://github.com/kubernetes/kubernetes/pull/67476\n\n### Kubelet metric changes\n\n#### Make metrics aggregatable\n\nCurrently, all Kubelet metrics are exposed as summary data types. This means that it is impossible to calculate certain metrics in aggregate across a cluster, as summaries cannot be aggregated meaningfully. For example, currently one cannot calculate the [pod start latency in a given percentile on a cluster](https://github.com/kubernetes/kubernetes/issues/66791).\n\nHence, where possible, we should change summaries to histograms, or provide histograms in addition to summaries like with the API server metrics.\n\nhttps://github.com/kubernetes/kubernetes/pull/72323\n\nhttps://github.com/kubernetes/kubernetes/pull/72470\n\nhttps://github.com/kubernetes/kubernetes/pull/73820\n\n#### Export less metrics\n\nhttps://github.com/kubernetes/kubernetes/issues/68522\n\n#### Prevent apiserver's metrics from accidental registration\n\nhttps://github.com/kubernetes/kubernetes/pull/63924\n\n#### Prober metrics\n\nMake prober metrics introduced in https://github.com/kubernetes/kubernetes/pull/61369 conform to the [Kubernetes instrumentation guidelines](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/instrumentation.md).\n\n### Kube-scheduler metric changes\n\nhttps://github.com/kubernetes/kubernetes/pull/72332\n\n### Kube-proxy metric changes\n\n#### Change proxy metrics to conform metrics guidelines\n\nhttps://github.com/kubernetes/kubernetes/pull/72334\n\n#### Clean the deprecated metrics which introduced in v1.14\n\nhttps://github.com/kubernetes/kubernetes/pull/75023\n\n### Kube-apiserver metric changes\n\n#### Apiserver and etcd metrics\n\nhttps://github.com/kubernetes/kubernetes/pull/72336\n\n#### Fix admission metrics in true units\n\nhttps://github.com/kubernetes/kubernetes/pull/72343\n\n#### Remove the deprecated admission metrics\n\nhttps://github.com/kubernetes/kubernetes/pull/75279\n\n### Client-go metric changes\n\n#### Workqueue metrics\n\nWorkqueue metrics need follow prometheus best practices and naming conventions.\n\n* Instead of naming metrics based on the name of the workqueue, create metrics for workqueues that use name as a label.\n* Use the recommended base units.\n* Change summaries to histograms.\n\nhttps://github.com/kubernetes/kubernetes/pull/71300\n\n### Convert latency/latencies in metrics name to duration\n\nhttps://github.com/kubernetes/kubernetes/pull/74418\n\n### Risks and Mitigations\n\nRisks include users upgrading Kubernetes, but not updating their usage of Kubernetes exposed metrics in alerting and dashboarding potentially causing incidents to go unnoticed.\n\nTo prevent this, we will implement recording rules for Prometheus that allow best effort backward compatibility as well as update uses of breaking metric usages in the [Kubernetes monitoring mixin](https://github.com/kubernetes-monitoring/kubernetes-mixin), a widely used collection of Prometheus alerts and Grafana dashboards for Kubernetes.\n\n## Deprecation Plan\n\nIn our efforts to change existing old metrics, we flag them `(Deprecated)` in the front of metrics help text.\n\nThese old metrics will be deprecated in v1.14 and coexist with the new replacement metrics. Users can use this release to change related monitoring rules and dashboards.\n\nThe release target of removing the deprecated metrics is v1.15.\n\n## Graduation Criteria\n\nAll metrics exposed by components from kubernetes/kubernetes follow Prometheus best practices and (nice to have) tooling is built and enabled in CI to prevent simple violations of said best practices.\n\n## Implementation History\n\nMultiple pull requests have already been opened, but not merged as of writing of this document.\n"
+  },
+  {
+    "id": "58d559d0fdb9999e17f7168204c0bdaf",
+    "title": "Event series API",
+    "authors": ["@gmarek"],
+    "owningSig": "sig-instrumentation",
+    "participatingSigs": ["sig-instrumentation", "sig-scalability", "sig-architecture"],
+    "reviewers": ["@wojtekt", "@bgrant0607"],
+    "approvers": ["@wojtekt", "@bgrant0607"],
+    "editor": "TBD",
+    "creationDate": "2019-01-31",
+    "lastUpdated": "2019-01-31",
+    "status": "implementable",
+    "seeAlso": ["n/a"],
+    "replaces": ["n/a"],
+    "supersededBy": ["n/a"],
+    "markdown": "\n# Title\n\nEvent series API\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Graduation Criteria\n\nBeta:\n\n- Ensure we do not have any performance regression compared to the orignal API\n- test coverage for edge-cases\n\n\nGA:\n\n- Update Event semantics such that they'll be considered useful by app developers\n- Reduce impact that Events have on the system's performance and stability\n- Switch all the controllers to use the new Event API\n\n## Implementation History\n\n- 2017-10-7 design proposal merged under [kubernetes/community](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/events-redesign.md)\n- 2017-11-23 Event API group is [merged](https://github.com/kubernetes/kubernetes/pull/49112)\n"
+  },
+  {
+    "id": "3bfec8929968e197a693142711c73ff3",
+    "title": "Kubernetes Control-Plane Metrics Stability",
+    "authors": ["@logicalhan"],
+    "owningSig": "sig-instrumentation",
+    "participatingSigs": ["sig-instrumentation", "sig-api-machinery", "sig-node"],
+    "reviewers": [
+      "@brancz",
+      "@x13n",
+      "@DirectXMan12",
+      "@lavalamp",
+      "@dashpole",
+      "@ehashman",
+      "@mml"
+    ],
+    "approvers": ["@brancz", "@x13n"],
+    "editor": "@brancz",
+    "creationDate": "2019-04-04",
+    "lastUpdated": "2019-11-06",
+    "status": "implementable",
+    "seeAlso": ["20181106-kubernetes-metrics-overhaul"],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Kubernetes Control-Plane Metrics Stability\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Background](#background)\n- [Proposal](#proposal)\n  - [Metric Definition Phase](#metric-definition-phase)\n  - [Metric Instantiation Phase](#metric-instantiation-phase)\n  - [Metric Enrollment Phase](#metric-enrollment-phase)\n- [Stability Classes](#stability-classes)\n- [API Review](#api-review)\n- [Deprecation Lifecycle](#deprecation-lifecycle)\n  - [Show Hidden Metrics](#show-hidden-metrics)\n  - [Why Not Bool Flag](#why-not-bool-flag)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n- [Drawbacks](#drawbacks)\n- [Alternatives](#alternatives)\n- [Unresolved Questions](#unresolved-questions)\n  - [Static Analysis for Validation](#static-analysis-for-validation)\n  - [Beta Stability Level](#beta-stability-level)\n  - [Prometheus Labels vs OpenCensus-type Tags](#prometheus-labels-vs-opencensus-type-tags)\n  - [Dynamically Registered Metrics](#dynamically-registered-metrics)\n- [Implementation History](#implementation-history)\n- [References](#references)\n    - [Metric Renaming](#metric-renaming)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nCurrently metrics emitted in the Kubernetes control-plane do not offer any stability guarantees. This Kubernetes Enhancement Proposal (KEP) proposes a strategy and framework for programmatically expressing how stable a metric is, i.e. whether a metric's name, type and [labels](https://prometheus.io/docs/practices/naming/#labels) (i.e. dimensions) is liable to change. Since this document will likely evolve with ongoing discussion around metric stability, it will be updated accordingly.\n\n## Motivation\n\nMetrics stability has been an ongoing community concern. Oftentimes, cluster monitoring infrastructure assumes the stability of at least some control-plane metrics; thus, it would be prudent to offer some sort of guarantees around control-plane metrics, treating it more properly as an API.\nSince the [metrics overhaul](https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20181106-kubernetes-metrics-overhaul.md) is nearing completion, there should be less reason to introduce breaking changes to metrics, making it an opportune time to introduce metric stability rules. Specifically, this KEP intends to address metric stability from an ingestion point of view.\n\nGuarantees around metrics have been [proposed previously](https://docs.google.com/document/d/1_CdNWIjPBqVDMvu82aJICQsSCbh2BR-y9a8uXjQm4TI/edit#) and there are [ongoing community discussions](https://groups.google.com/forum/#!topic/kubernetes-sig-instrumentation/XbElxDtww0Y) around this issue. Some suggested solutions include:\n\n1. Having a ‘stable’ metrics endpoint, i.e. ‘/metrics/v1’\n2. Leaving metrics as is and documenting the ones which have a stability guarantee\n\nThis KEP suggests another alternative but is very much in line with the spirit of the other proposed solutions.\n\n### Goals\n\n * Describe the various stability guarantees for the consumption of control-plane metrics.\n * Define a uniform mechanism for expressing metric stability.\n\n### Non-Goals\n\n* We are __*not*__ defining which specific control-plane metrics are actually stable.\n* We are __*not*__ providing guarantees around specific values in metrics; as such, breakages in alerting based of off assumptions on specific values in metrics are out-of-scope.\n* Defining the precise mechanism by which we will validate metric rule violations (i.e. static analysis) is also __*not*__ in scope.\n\n## Background\n\nKubernetes control-plane binaries (i.e. scheduler, kubelet, controller-manager, apiserver) use a Prometheus client to export binary-specific metrics to a ‘/metrics’ endpoint in Prometheus format. Metrics are first defined and then instantiated; later they are registered to a metrics registry. The http handler for the metrics endpoint then delegates responses to the underlying registry.\n\nFor the remainder of this document, I will refer to the following terms by these definitions:\n\n* __metric definition__ - this refers to defining a metric. In Kubernetes, we use the standard Prometheus pattern of using an options struct to define name, type, description of a metric.\n* __metric instantiation__ - this refers to creating an instance of a metric. A metric definition is passed into a metric constructor which, in Kubernetes, is a Prometheus metric constructor (example).\n* __metric enrollment__ - after being defined and created, individual metrics are officially enrolled to a metrics registry (currently a global one).\n* __metric registration process__ - I use this to refer to the entire lifecycle of a metric from definition, to instantiation, then enrollment.\n\nThe fact that the metric registration process always involves these steps is significant because it allows for the possibility of injecting custom behavior in and around these steps.\n\n## Proposal\n\nThis KEP proposes a programmatic mechanism to express the stability of a given control plane metric. Individual metrics would be quasi-versioned, i.e. they would have additional bits of metadata which would indicate whether that metric was alpha (not-stable), stable, or deprecated. Metric stability guarantees would depend on the values of those additional bits. This information would be represented to users in two primary ways: (1) metric stability information would be automatically to individual metrics help descriptions and (2) in the case of deprecated metrics, warning logs would be emitted on metric registration.\n\nSpecifically, this would involve injecting custom behavior during metric registration process by wrapping metric definition, instantiation and enrollment.\n\n### Metric Definition Phase\n\nCurrently, the metric definition phase looks like this:\n\n```go\nvar someMetricDefinition = prometheus.CounterOpts{\n    Name: \"some_metric\",\n    Help: \"some description\",\n}\n```\n\nSince we are using the Prometheus provided struct, we are constrained to Prometheus provided fields. However, using a custom struct affords us the following:\n```go\nvar deprecatedMetricDefinition = kubemetrics.CounterOpts{\n    Name: \"some_deprecated_metric\",\n    Help: \"some description\",\n    StabilityLevel: kubemetrics.STABLE, // this is also a custom metadata field\n    DeprecatedVersion: \"1.15\", // this is a custom metadata field\n}\n\nvar alphaMetricDefinition = kubemetrics.CounterOpts{\n    Name: \"some_alpha_metric\",\n    Help: \"some description\",\n    StabilityLevel: kubemetrics.ALPHA, // this is also a custom metadata field\n\tDeprecatedVersion: \"1.15\", // this can optionally be included on alpha metrics, although there is no change to contractual stability guarantees\n}\n```\n\n### Metric Instantiation Phase\n\nCurrently, the metric instantiation phase looks like this:\n\n```go\nvar someCounterVecMetric = prometheus.NewCounterVec(\n    someMetricDefinition,\n    []string{\"some-label\", \"other-label\"},\n}\n```\n\nWrapping the Prometheus constructors would allow us to take, as inputs, the modified metric definitions defined above, returning a custom Kubernetes metric object which contains the metric which would have been instantiated as well as the custom metadata:\n\n```go\nvar deprecatedMetric = kubemetrics.NewCounterVec( // this is a wrapped initializer, which takes in our custom metric definitions\n    deprecatedMetricDefinition, // this is our custom wrapped metric definition from above\n    []string{\"some-label\", \"other-label\"},\n}\nvar alphaMetric = kubemetrics.NewCounterVec{\n    alphaMetricDefinition, // this is also our custom wrapped metric definition from above\n    []string{\"some-label\", \"other-label\"},\n}\n```\n\n### Metric Enrollment Phase\n\nCurrently, metric enrollment involves calls to a Prometheus function which enrolls the metric in a global registry, like so:\n```go\nprometheus.MustRegister(someCounterVecMetric)\n```\n\nWrapping a prometheus registry with a Kubernetes specific one, would allow us to take our custom metrics from our instantiation phase and execute custom logic based on our custom metadata. Our custom registry would hold a reference to a prometheus registry and defer metric enrollment unless preconditions were met:\n\n```go\nimport version \"k8s.io/apimachinery/pkg/version\"\n\ntype Registry struct {\n    promregistry *prometheus.Registry\n    KubeVersion version.Info\n}\n\n// inject custom registration behavior into our registry wrapper\nfunc (r *Registry) MustRegister(metric kubemetrics.Metric) {\n    // pretend we have a version comparison utility library\n    if metricutils.compare(metric.DeprecatedVersion).isLessThan(r.KubeVersion) {\n        // check if binary has deprecated metrics enabled otherwise\n        // no-op registration\n        return\n    } else if metricutils.compare(metric.DeprecatedVersion).isEqual(r.KubeVersion) {\n        // append deprecated text to description\n        // emit warning in logs\n        // continue to actual registration\n    }\n    // append alpha text to metric description if metric.isAlpha\n    // fallback to original prometheus behavior\n    r.promregistry.MustRegister(metric.realMetric)\n}\n\n```\n\nWhich we would invoke, like so:\n```go\nkubemetrics.MustRegister(deprecatedMetric)\nkubemetrics.MustRegister(alphaMetric)\n```\n\n## Stability Classes\n\nThis proposal introduces two stability classes for metrics: (1) Alpha, (2) Stable. These classes are intended to make explicit the API contract between the control-plane and the consumer of control-plane metrics.\n\n__Alpha__ metrics have __*no*__ stability guarantees; as such they can be modified or deleted at any time. At this time, all Kubernetes metrics implicitly fall into this category.\n\n__Stable__ metrics can be guaranteed to *not change*, except that the metric may become marked deprecated for a future Kubernetes version. By *not change*, we mean three things:\n\n1. the metric itself will not be deleted ([or renamed](#metric-renaming))\n2. the type of metric will not be modified\n3. no labels can be added or removed from this metric\n\nFrom an ingestion point of view, it is backwards-compatible to add or remove possible __values__ for labels which already do exist (but __not__ labels themselves). Therefore, adding or removing __values__ from an existing label is permissible. Stable metrics can also be marked as __deprecated__ for a future Kubernetes version, since this is a metadata field and does not actually change the metric itself.\n\nRemoving or adding labels from stable metrics is not permissible. In order to add/remove a label to an existing stable metric, one would have to introduce a new metric and deprecate the stable one; otherwise this would violate compatibility agreements.\n\nAs an aside, all metrics should be able to be individually disabled by the cluster administrator, regardless of stability class. By default, all non-deprecated metrics will be automatically registered to the metrics endpoint unless explicitly blacklisted via a command line flag (i.e. '--disable-metrics=somebrokenmetric,anothermetric').\n\n## API Review\n\nGraduating a metric to a stable state is a contractual API agreement, as such, it would be desirable to require an api-review (to sig-instrumentation) for graduating or deprecating a metric (in line with current Kubernetes [api-review processes](https://github.com/kubernetes/community/blob/master/sig-architecture/api-review-process.md)). However, initiating or mandating such API review has historically been problematic for sig-instrumentation since, while a horizontal effort, is not automatically added as reviewers for metrics related changes.\n\nOne possible solution is through something similar to the existing Kubernetes conformance test gates (thanks @liggitt for pointing this one out). We will have a script which can generate a list of the current stable metrics (via static analysis). This list will be checked in. During the CI/CD flow, a verify script will run to generate a new list of stable metrics. If there is a diff present, then the verify script will fail, since the file should be updated and checked in. Thus, the file must be checked in and since the file will live in a directory owned by sig-instrumentation, sig-instrumentation approval on that PR will be required.\n\n## Deprecation Lifecycle\n\nThis proposal introduces deprecation metadata for metrics, to be used to define a deprecation lifecycle. Metrics can be annotated with a Kubernetes version, from which point that metric will be considered deprecated. This allows us to indicate that a metric is slated for future removal and provides the consumer a reasonable window in which they can make changes to their monitoring infrastructure which depends on this metric.\n\nWhile deprecation policies only actually change stability guarantees for __stable__ metrics (and not __alpha__ ones), deprecation information may however be optionally provided on alpha metrics to help component owners inform users of future intent, to help with transition plans (this change was made at the request of @dashpole, who helpfully pointed out that it would be nice to be able signal future intent even for alpha metrics).\n\nWhen a stable metric undergoes the deprecation process, we are signaling that the metric will eventually be deleted. The lifecyle looks roughly like this (each stage represents a Kubernetes release):\n\n__Stable metric__ -\u003e __Deprecated metric__ -\u003e __Hidden metric__ -\u003e __Deletion__\n\n__Deprecated__ metrics have the same stability guarantees of their counterparts. If a stable metric is deprecated, then a deprecated stable metric is guaranteed to *not change*. When deprecating a stable metric, a future Kubernetes release is specified as the point from which the metric will be considered deprecated.\n\n```go\nvar someCounter = kubemetrics.CounterOpts{\n    Name: \"some_counter\",\n    Help: \"this counts things\",\n    StabilityLevel: kubemetrics.STABLE,\n    DeprecatedVersion: \"1.15\", // this metric is deprecated when the Kubernetes version == 1.15\n}\n````\n\n__Deprecated__ metrics will have their description text prefixed with a deprecation notice string '(Deprecated from x.y)' and a warning log will be emitted during metric registration (in the spirit of the official [Kubernetes deprecation policy](https://kubernetes.io/docs/reference/using-api/deprecation-policy/#deprecating-a-flag-or-cli)).\n\nBefore deprecation:\n\n```text\n# HELP some_counter this counts things\n# TYPE some_counter counter\nsome_counter 0\n```\n\nDuring deprecation:\n```text\n# HELP some_counter (Deprecated from 1.15) this counts things\n# TYPE some_counter counter\nsome_counter 0\n```\nLike their stable metric counterparts, deprecated metrics will be automatically registered to the metrics endpoint.\n\nOn a subsequent release (when the metric's deprecatedVersion is equal to current_kubernetes_version - 1)), a deprecated metric will become a __hidden metric__. _Unlike_ their deprecated counterparts, hidden metrics will __*no longer be automatically registered*__ to the metrics endpoint (hence hidden). However, they can be explicitly enabled through a command line flag on the binary (i.e. '--show-hidden-metrics-for-version=\u003cprevious minor release\u003e'). This is to provide cluster admins an escape hatch to properly migrate off of a deprecated metric, if they were not able to react to the earlier deprecation warnings. Hidden metrics should be deleted after one release.\n\n### Show Hidden Metrics\nAs described above, admins can enable hidden metrics through a command-line flag on a specific binary. \nThis intends to be used as an escape hatch for admins if they missed the migration of the metrics deprecated in the last release. \n\nThe flag `show-hidden-metrics-for-version` takes a version for which you want to show metrics deprecated in that release.\nThe version is expressed as __x.y__, where __x__ is the major version, __y__ is the minor version.\nThe patch version is not needed even though a metrics can be deprecated in a patch release, the reason for that is\nthe metrics deprecation policy runs against the minor release.\n\nThe flag can only take the previous minor version as it's value. \nAll metrics hidden in previous will be emitted if admins set the previous version to `show-hidden-metrics-for-version`.\nThe too old version is not allowed because this violates the metrics deprecated policy.\n\nTake metric `A` as an example, here assumed that `A` is deprecated in `1.n`.\nAccording to metrics deprecated policy, we can reach the following conclusion:\n- In release `1.n`, the metric is deprecated, and it can be emitted by default.\n- In release `1.n+1`, the metric is hidden by default and it can be emitted by command line `show-hidden-metrics-for-version=1.n`.\n- In release `1.n+2`, the metric should be removed from the codebase. No escape hatch anymore.\n\nSo, if admins want to enable metric `A` in release `1.n+1`, they should set `1.n` to the command line flag. \nThat is `show-hidden-metrics=1.n`. \n\n### Why Not Bool Flag\nAlternatively, another solution which was previously suggested(refer to the discussion on [PR](https://github.com/kubernetes/kubernetes/pull/84292)) \nwas provide a bool flag-like `show-hidden-metrics`. That works like:\n- `show-hidden-metrics=true`: enable all hidden metrics deprecated in a previous minor version.\n- `show-hidden-metrics=false`: the default value, do nothing.\n\nThis proposal has a side effect(thanks for @lavalamp pointed it out) in the scenario:\n1. in version X, turn this on to get back deprecated metric M\n2. in version Y, not turn this off, and therefore fail to notice metric N is being deprecated\n3. in version Z, metric N is removed with (effectively) no warning\n\n## Design Details\n\n### Test Plan\n\nInternal wrappers can be unit tested. There has been some discussion around providing APIs around metric definitions, hopefully with the end goal of being able to augment our test strategy with current and historical metric definition data.\n\n### Graduation Criteria\n\nThis feature enhancement will not require graduation criteria. We intend to isolate these changes into a sig-instrumentation repo, build out the framework and the static analysis validation piece (which will require it's own KEP) and then have another KEP which details migration strategy across the kubernetes codebase. Since this can be built in place without affecting any surrounding code, it is safe.\n\n## Drawbacks\n\nMore generally, this proposal has the drawbacks which any proposal suggesting a more rigorous enforcement of an API is going to have. There is always a tradeoff between the ease at which a developer can make breaking changes to an API, with consumers' ability to reliably use that API.\n\nRelative to a more hands-off approach, like one where we just document the metrics which the community has agreed to 'certify' as stable, this approach is definitely more heavyweight. This approach involves more code and more code is more maintenance. However, most of the code will be centralized and the internal logic is easily unit-testable. We also do not have to worry much about changing internal API semantics, since our wrappers will be used internally only, which means it should be easy to modify for new usecases in the future. This sort of approach also enables static analysis tooling around metrics which we could run in precommit.\n\nAlso, we should note that this approach can be manufactured in-place; this framework could be rolled out without actually introducing any backwards-incompatible changes (unlike moving stable metrics to a '/metrics/v1' endpoint).\n\nThere is also some inflexibility in responding to the situation where code is re-architected in such a way that it's no longer feasible to provide a metric (e.g. there's no longer anything to measure). Generally, we would want to try to avoid this situation by not making a metric stable if there's any way for it to get refactored away. Currently, in this sort of case, the metrics stability proposal would only dictate that we continue to register the metric and undergo the normal metric deprecation policy, as it would be necessary for avoiding ingestion pipeline breakages (thanks @DirectXMan12 for pointing this out).\n\n## Alternatives\n\nUsing a more traditional versioned endpoint was one of the first suggested ideas. However, metrics basically form a single API group so making a change to a single (previously considered stable) metric would necessitate a version bump for all metrics. In the worst case, version bumps for metrics could occur with each release, which is undesirable from a consumption point of view.\n\nIt would also be possible to group metrics into distinct endpoints, in order to avoid global version bumps. However, this breaks the more common metrics ingestion patterns, i.e. as a consumer of metrics for a component, you would no longer be able to assume all of your relevant metrics come from one location. This is also a potentially confusing pattern for consumer of metrics, since you would have to manage a series of metrics for a given component and also be cognizant of the version for each of these components. It would be easy to get wrong.\n\nAlternatively, one lightweight solution which was previously suggested was documenting the metrics which have stability guarantees. However, this is prone to documentation rot and adds manual (and error-prone) overhead to the metrics process.\n\n## Unresolved Questions\n\n### Static Analysis for Validation\n\nHaving a set of wrappers in place which allows us to define custom metadata on metrics is quite powerful, since it enables a number of _theorectically possible_ features, like static analysis for verifying metric guarantees during a precommit phase. How we would actually go about doing this is TBD. It is possible to use the metrics registry to output metric descriptions in a separate endpoint; using static analysis we could validate metrics descriptions with our stability rules.\n\n### Beta Stability Level\n\nCurrently I am inclined to omit the beta stage from metric versioning if only to reduce initial complexity. It may however become more desirable to include this state in a later design/implementation phase.\n\n### Prometheus Labels vs OpenCensus-type Tags\n\nHaving these series of wrappers in place allows us to potentially provide a custom wrapper struct around Prometheus labels. This is particularly desirable because labels are shared across metrics and we may want to define uniform behavior for a given label ([constraining values for labels](https://github.com/kubernetes/kubernetes/issues/75839#issuecomment-478654080), [whitelisting values for a label](https://github.com/kubernetes/kubernetes/issues/76302)). Prometheus labels are pretty primitive (i.e. lists of strings) but potentially we may want an abstraction which more closely resembles [open-census tags](https://opencensus.io/tag/).\n\n### Dynamically Registered Metrics\n\nMetrics which are added dynamically after application boot - Metrics which are dynamically added depending on things which occur during runtime should probably not be allowed to be considered stable metrics, since we can't rely on them to exist reliably.\n\n## Implementation History\n\n- Wrappers added around prometheus registry and counter/counterVec metric type [PR-77037](https://github.com/kubernetes/kubernetes/pull/77037)\n- Wrappers added around prometheus gauge, histogram \u0026 summary metric types [PR-77618](https://github.com/kubernetes/kubernetes/pull/77618)\n- Make sig-instrumentation owners for component-base metrics directory [PR-77621](https://github.com/kubernetes/kubernetes/pull/77621)\n\n## References\n\n#### Metric Renaming\n\nMetric renaming is be tantamount to deleting a metric and introducing a new one. Accordingly, metric renaming will also be disallowed for stable metrics.\n"
+  },
+  {
+    "id": "42344d1a00d36baa9f3868bfeb1de041",
+    "title": "Watch support for metrics APIs",
+    "authors": ["@x13n"],
+    "owningSig": "sig-instrumentation",
+    "participatingSigs": ["sig-autoscaling"],
+    "reviewers": ["TBD"],
+    "approvers": ["TBD"],
+    "editor": "TBD",
+    "creationDate": "2019-04-25",
+    "lastUpdated": "2019-04-29",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Metrics API watch support\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n  - [Summary](#summary)\n  - [Motivation](#motivation)\n    - [Goals](#goals)\n    - [Non-Goals](#non-goals)\n  - [Proposal](#proposal)\n    - [User Stories](#user-stories)\n      - [HPA](#hpa)\n      - [Custom metrics provider](#custom-metrics-provider)\n    - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n    - [Risks and Mitigations](#risks-and-mitigations)\n  - [Design Details](#design-details)\n    - [Test Plan](#test-plan)\n    - [Graduation Criteria](#graduation-criteria)\n    - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n    - [Version Skew Strategy](#version-skew-strategy)\n  - [Implementation History](#implementation-history)\n  - [Drawbacks [optional]](#drawbacks-optional)\n- [Related resources](#related-resources)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nProvide watch capability to all resource metrics APIs: `metrics.k8s.io`,\n`custom.metrics.k8s.io` and `external.metrics.k8s.io`, [similarly to regular\nKubernetes APIs](https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes).\n\n## Motivation\n\nThere are three APIs for serving metrics. All of them support reading the\nmetrics in a request-response manner, forcing the clients (e.g. HPA) interested\nin up-to-date values to poll in a loop. This introduces additional latency:\nbetween the time when new metric values are known to the process serving the\nAPI and the time when a client interested in reading them actually fetches the\ndata.\n\n### Goals\n\n- Allow resource metrics clients to subscribe to stream metric changes.\n\n### Non-Goals\n\n- Graduate the APIs to GA. This needs to be done eventually, but is out of scope\n  for this work.\n\n## Proposal\n\nThis proposal is essentially about implementing [Efficient detection of\nchanges](https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes)\nfor metrics. GET requests will accept additional `watch` parameter, which would\ncause the API to start streaming results. Since old metric values are never\nmodified, the only supported update type will be `ADDED`, when a new data point\nappears. Metrics don't contain any resourceVersion associated with them, so it\nwon't be possible to retrieve old values by passing `resourceVersion` parameter.\nInstead, this parameter will be ignored and all recent data points will be\nreturned instead. This means metrics APIs will never return `410 Gone` error\ncode.\n\n### User Stories\n\nThere are two sides to that proposal: the API producers and consumers. Examples\nbelow include one consumer (HPA) and one hypothetical producer.\n\n#### HPA\n\nAs an autoscaling solution, I will be able to subscribe to updates on a certain\nlabelSelector and get new metrics as soon as they are known to the metric\nbackend.\n\n#### Custom metrics provider\n\nAs a metrics provider, I will be able to provide a low-latency Metrics API\nimplementation by taking advantage of backend specific features (e.g. streaming\nAPIs or known best polling interval).\n\n### Implementation Details/Notes/Constraints [optional]\n\nTBD:\nWhat are the caveats to the implementation?\nWhat are some important details that didn't come across above.\nGo in to as much detail as necessary here.\nThis might be a good place to talk about core concepts and how they releate.\n\n### Risks and Mitigations\n\nTBD: preventing producers/consumers from breaking due to the API version change.\n\n## Design Details\n\n### Test Plan\n\nTBD.\n\n**Note:** *Section not required until targeted at a release.*\n\nConsider the following in developing a test plan for this enhancement:\n- Will there be e2e and integration tests, in addition to unit tests?\n- How will it be tested in isolation vs with other components?\n\nNo need to outline all of the test cases, just the general strategy.\nAnything that would count as tricky in the implementation and anything particularly challenging to test should be called out.\n\nAll code is expected to have adequate tests (eventually with coverage expectations).\nPlease adhere to the [Kubernetes testing guidelines][testing-guidelines] when drafting this test plan.\n\n[testing-guidelines]: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md\n\n### Graduation Criteria\n\nSince the APIs already exist in beta and the change is backwards-compatible,\nthis proposal will be applied to the beta API, updating it from v1beta$(n) to\nv1beta$(n+1).\n\nStability of this feature proven by at least one backend implementation for\n`metrics.k8s.io` and `custom.metrics.k8s.io` will be a blocker for graduating\nthese APIs to v1.\n\nThis stability will be measured by e2e tests that will fetch the data using\nwatch.\n\n### Upgrade / Downgrade Strategy\n\nTBD.\n\nIf applicable, how will the component be upgraded and downgraded? Make sure this is in the test plan.\n\nConsider the following in developing an upgrade/downgrade strategy for this enhancement:\n- What changes (in invocations, configurations, API use, etc.) is an existing cluster required to make on upgrade in order to keep previous behavior?\n- What changes (in invocations, configurations, API use, etc.) is an existing cluster required to make on upgrade in order to make use of the enhancement?\n\n### Version Skew Strategy\n\nTBD.\n\nIf applicable, how will the component handle version skew with other components? What are the guarantees? Make sure\nthis is in the test plan.\n\nConsider the following in developing a version skew strategy for this enhancement:\n- Does this enhancement involve coordinating behavior in the control plane and in the kubelet? How does an n-2 kubelet without this feature available behave when this feature is used?\n- Will any other components on the node change? For example, changes to CSI, CRI or CNI may require updating that component before the kubelet.\n\n## Implementation History\n\nMajor milestones in the life cycle of a KEP should be tracked in `Implementation History`.\nMajor milestones might include\n\n- the `Summary` and `Motivation` sections being merged signaling SIG acceptance\n- the `Proposal` section being merged signaling agreement on a proposed design\n- the date implementation started\n- the first Kubernetes release where an initial version of the KEP was available\n- the version of Kubernetes where the KEP graduated to general availability\n- when the KEP was retired or superseded\n\n## Drawbacks [optional]\n\nNo custom metrics backend today offers a streaming API that would allow a\nstraightforward implementation of the watch. However, the fact that Kubernetes\nmetrics APIs will support streaming with watch may encourage some backends to\nadd such support. Additionally, the polling frequency will be specific to\nrelevant adapters, rather than to the metrics client.\n\n# Related resources\n\nSIG instrumentation discussions:\n- [Custom/External Metrics API watch](https://groups.google.com/forum/#!topic/kubernetes-sig-instrumentation/nJvDyIwDgu8)\n- [Resource Metrics API watch](https://groups.google.com/d/msg/kubernetes-sig-instrumentation/_b6c0oyPLJA/Y4rMQTBDAgAJ)\n"
+  },
+  {
+    "id": "1b233aa2bebb28fb1213813986323eb7",
+    "title": "Metrics Stability Migration",
+    "authors": ["@logicalhan", "@solodov"],
+    "owningSig": "sig-instrumentation",
+    "participatingSigs": [
+      "sig-scheduling",
+      "sig-node",
+      "sig-api-machinery",
+      "sig-cluster-lifecycle",
+      "sig-cloud-provider"
+    ],
+    "reviewers": [
+      "@brancz from sig-instrumentation",
+      "@dashpole from sig-node",
+      "@sttts from sig-api-machinery",
+      "@DirectXMan12 from sig-cluster-lifecycle",
+      "@bsalamat from sig-scheduling",
+      "@andrewsykim from sig-cloud-provider"
+    ],
+    "approvers": ["@brancz"],
+    "editor": "",
+    "creationDate": "2019-06-05",
+    "lastUpdated": "2019-06-27",
+    "status": "implementable",
+    "seeAlso": [
+      "20181106-kubernetes-metrics-overhaul",
+      "20190404-kubernetes-control-plane-metrics-stability"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Metrics Stability Migration\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [General Migration Strategy](#general-migration-strategy)\n  - [Shared metrics](#shared-metrics)\n  - [Deprecation of modified metrics from metrics overhaul KEP](#deprecation-of-modified-metrics-from-metrics-overhaul-kep)\n- [Implementation History](#implementation-history)\n- [Graduation Criteria](#graduation-criteria)\n- [Testing Plan](#testing-plan)\n\u003c!-- /toc --\u003e\n\n\n## Summary\n\nThis KEP intends to document and communicate the general strategy for migrating the control-plane metrics stability framework across the Kubernetes codebase. Most of the framework design decisions have been determined and outlined in [an earlier KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md).\n\n## Motivation\n\nWe want to start using the metrics stability framework built based off the [Kubernetes Control-Plane Metrics Stability KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md), so that we can define stability levels for metrics in the Kubernetes codebase. These stability levels would provide API compatibility guarantees across version bumps.\n\n### Goals\n\n * Outline the general strategy for migrating metrics to the stability framework\n * Explicitly define a strategy for handling migration of shared metrics.\n * Maintain backwards compatibility through the migration path. *This excludes metrics which are slated to be removed after deprecation due to metrics overhaul.*\n * Communicate upcoming changes to metrics to respective component owners.\n\n### Non-Goals\n\n* During migration, we will __*not*__ be determining whether a metric is considered stable. Any metrics which will be promoted to a stable status must be done post-migration.\n* Kubelet '/metrics/resource' and '/metrics/cadvisor' are out of scope for this migration due to use of non-standard prometheus collectors.\n\n## General Migration Strategy\n\nTo keep migration PRs limited in scope (i.e. ideally belonging to a single component owner at a time), we prefer to approach migration in a piecemeal fashion. Each metrics endpoint can be considered an atomic unit of migration. This will allow us to avoid migrating the entire codebase at once.\n\nKubernetes control-plane binaries can expose one or more metrics endpoints:\n\n* one controller-manager metrics endpoint\n    * [/metrics](https://github.com/kubernetes/kubernetes/blob/release-1.15/cmd/controller-manager/app/serve.go#L65)\n* one kube-proxy metrics endpoint\n    * [/metrics](https://github.com/kubernetes/kubernetes/blob/release-1.15/cmd/kube-proxy/app/server.go#L570)\n* one kube-apiserver metrics endpoint\n    * [/metrics](https://github.com/kubernetes/kubernetes/blob/release-1.15/staging/src/k8s.io/apiserver/pkg/server/routes/metrics.go#L36)\n* one scheduler metrics endpoint\n    * [/metrics](https://github.com/kubernetes/kubernetes/blob/release-1.15/cmd/kube-scheduler/app/server.go#L289)\n* [four kubelet metrics endpoints](https://github.com/kubernetes/kubernetes/blob/release-1.15/staging/src/k8s.io/apiserver/pkg/server/routes/metrics.go#L36)\n    * [/metrics](https://github.com/kubernetes/kubernetes/blob/release-1.15/pkg/kubelet/server/server.go#L299)\n    * [/metrics/probes](https://github.com/kubernetes/kubernetes/blob/release-1.15/pkg/kubelet/server/server.go#L329-L331)\n    * (out of scope) ~~[/metrics/cadvisor](https://github.com/kubernetes/kubernetes/blob/release-1.15/pkg/kubelet/server/server.go#L315)~~\n    * (out of scope) ~~[/metrics/resource](https://github.com/kubernetes/kubernetes/blob/release-1.15/pkg/kubelet/server/server.go#L321-L323)~~\n\nFollowing our desired approach, each of metrics endpoint above (except those out of scope) will be accompanied by an individual PR for migrating that endpoint.\n\n### Shared metrics\n\nShared metrics makes piecemeal migration potentially difficult because metrics in shared code will either be migrated or not, and a component which uses the shared metric can be in the opposite state. Currently, there are groups of metrics which are __*shared*__ across binaries:\n\n1. [Kubernetes build info metadata metric](https://github.com/kubernetes/kubernetes/blob/release-1.15/pkg/version/prometheus/prometheus.go#L26-L38) - used by kube-apiserver, controller-manager, hollow-node, kubelet, kube-proxy, scheduler.\n2. [Client-go metrics](https://github.com/kubernetes/kubernetes/blob/release-1.15/pkg/util/prometheusclientgo/adapters.go#L20-L24)\n    * [client metrics](https://github.com/kubernetes/kubernetes/blob/release-1.15/pkg/client/metrics/prometheus/prometheus.go#L61-L66)\n    * [leader-election metrics](https://github.com/kubernetes/kubernetes/blob/release-1.15/pkg/util/prometheusclientgo/leaderelection/adapter.go#L27-L29)\n    * [workqueue metrics](https://github.com/kubernetes/kubernetes/blob/release-1.15/pkg/util/workqueue/prometheus/prometheus.go)\n\nOur strategy around shared metrics is to simply duplicate shared metric files and create a version of the shared metrics which uses the new registries. When we migrate an endpoint, we can remove the import statements to the old file and replace it with references to our migrated variant. This has the additional benefit that once migration is complete, we can just delete the old variants of shared metrics.\n\n### Deprecation of modified metrics from metrics overhaul KEP\n\nThe [metrics overhaul KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20181106-kubernetes-metrics-overhaul.md) deprecates a number of metrics across the Kubernetes code-base. As a part of this migration, these metrics will be marked as deprecated since 1.16, meaning that they will be hidden automatically for 1.17. These metrics will be in a deprecated but 'hidden' state and will be able to be optionally enabled through command line flags for one release cycle, and they will be permanently deleted in 1.18.\n\n## Implementation History\n\n- [x] [Migrate kubelet metrics to use standard prometheus collectors](https://github.com/kubernetes/kubernetes/issues/79286)\n- [ ] Create migrated variants of shared client metrics\n- [ ] Create migrated variants of shared leader-election metrics\n- [ ] Create migrated variants of shared workqueue metrics\n- [ ] Migrate kubelet's /metrics/probes endpoint\n- [ ] Migrate apiserver /metrics endpoint\n- [ ] Migrate scheduler /metrics endpoint\n- [ ] Migrate kube-proxy /metrics endpoint\n- [ ] Migrate controller-manager /metrics endpoint (this include in-tree cloud-provider metrics)\n\nTBD (since this is not yet implemented)\n\n## Graduation Criteria\n\n- [ ] Prior to migrating a component, automated static analysis testing is in place to validate and verify API guarantees.\n- [ ] Adequate documentation exists for new flags on components.\n- [ ] Update instrumentation documents to reflect changes\n\n## Testing Plan\n\n- [ ] Prior to migrating a metric's endpoint, we will run local tests to verify that the same metrics are populated\n- [ ] All metrics framework code will have unit/integration tests\n- [ ] All validation and verification code will have unit/integration tests\n"
+  },
+  {
+    "id": "14746bd77921ce3f8f792b78b5a0dc92",
+    "title": "Metrics Validation and Verification",
+    "authors": ["@serathius", "@solodov", "@logicalhan"],
+    "owningSig": "sig-instrumentation",
+    "participatingSigs": ["sig-instrumentation", "sig-testing"],
+    "reviewers": ["@brancz"],
+    "approvers": ["@brancz"],
+    "editor": "TBD",
+    "creationDate": "2019-06-05",
+    "lastUpdated": "2019-06-05",
+    "status": "implementable",
+    "seeAlso": [
+      "20190404-kubernetes-control-plane-metrics-stability",
+      "20190605-metrics-stability-migration"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n # Metrics Validation and Verification\n\n ## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n- [Design Details](#design-details)\n  - [Static Analysis](#static-analysis)\n  - [Discovery of golang source files](#discovery-of-golang-source-files)\n  - [Format of defining stable metrics](#format-of-defining-stable-metrics)\n  - [Stable metric detection strategy](#stable-metric-detection-strategy)\n  - [Format of stable metrics list file](#format-of-stable-metrics-list-file)\n  - [Failure Modes](#failure-modes)\n  - [Performance](#performance)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Test Plan](#test-plan)\n- [Unresolved questions](#unresolved-questions)\n  - [Should code snippets used for unit testing be kept in separate files instead of strings?](#should-code-snippets-used-for-unit-testing-be-kept-in-separate-files-instead-of-strings)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis Kubernetes Enhancement Proposal (KEP) builds off of the framework proposed\nin the [Kubernetes Control-Plane Metrics Stability KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md)\nand proposes a strategy for ensuring conformance of metrics with official\nstability guarantees.\n\n## Motivation\n\nWhile the [Kubernetes Control Plane metrics stability KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md)\nprovides a framework to define stability levels for control-plane metrics,\nit does not provide a strategy for verifying and validating conformance to stated guarantees.\nThis KEP intends to propose a framework for validating and verifying metric guarantees.\n\n### Goals\n\n* Given a stable metric, validate that we cannot remove or modify it (other than adding deprecation information).\n* Given a deprecated but stable metric, validate that we cannot remove or modify it until the deprecation period has elapsed.\n* Given an alpha metric which is promoted to be 'stable', automatically include proper instrumentation reviewers\n  (for schema validation and conformance to metrics guidelines).\n* [Stretch] Provide a single source of stable metric documentation (thanks @ehashman for proposing it)\n\n### Non-Goals\n\n* Conformance testing will not apply to alpha metrics, since alpha metrics do not have stability guarantees.\n\n## Proposal\n\nWe will provide validation for metrics under the [new framework](https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md) with static analysis.\n\n## Design Details\n\nStable metrics testing will work in a similar (but not identical) fashion to the generic Kubernetes conformance tests.\nSig-instrumentation will own a directory under `test/instrumentation`.\nThere will be a subdirectory `testdata` in which a file `stable-metrics-list.txt` will live.\nThis file will be owned by sig-instrumentation.\nMetrics conformance tests will involve a static analysis script which will traverse the entire codebase and look for\nmetrics which are annotated as 'stable'.\nFor each stable metrics, this script will generate a stringified version of metric metadata (i.e. name, type, labels)\nwhich will then be appended together in lexographic order. This will be the output of this script.\n\nWe will add a pre-submit check, which will run in our CI pipeline, which will run our script with the current changes\nand compare that to existing, committed file. If there is a difference, the pre-submit check will fail.\nIn order to pass the pre-submit check, the original submitter of the PR will have to run a script `\ntest/instrumentation/update-stable-metrics.sh` which will run our static analysis code and overwrite `stable-metrics-list.yaml`.\nThis will cause `sig-instrumentation` to be tagged for approvals on the PR (since they own that file).\n \n \n### Static Analysis\n \nSimilarly to conformance test proposed analysis will be performed on golang source code using default abstract syntax tree parser `go/ast`.\n Handling all possible cases for how metrics can be instantiated would require executing the code itself and is not practical.\n To reduce number of cases needed handled we will be making assumption of non malicious intent of contributors.\n There are just too many ways one golang struct can be instantiated that would hide potential stable metrics.\n To ensure correctness and code simplicity we will be restricting how stable metrics can be declared to one format described below.\n Alpha metrics will not be analyzed as they don't have any stability guarantees.\n Their declaration will not have any restrictions, thus allowing dynamic generation.\n\n### Discovery of golang source files\n\nStable metric list will be a bazel genrule provided with locations of files from `//:all-src` target.\nThis target is auto-managed, validate in CI by `./hack/validate-bazel.sh` and should include all files in repository.\n Golang source files will be filtered by extension.\n\nList of skipped directories:\n* `vendor/` - Kubernetes metrics from external repos will be shared metrics that will be defined in k/k and injected as\n              dependency during registration\n\n### Format of defining stable metrics\n\nTo simplify static analysis implementation and reduce chance of missing metrics, we will restrict how stable metrics can be defined.\nStable metrics will use exactly the same functions as normal ones, but code defining them will need to comply to the specific format:\n\n* Metric options (e.g. `CounterOpts`) needs to be directly passed to function (e.g. 1`kubemetrics.NewCounterVec`).\n* Metric arguments can only be set to values. Fields cannot be set to const, variables or function result (apart of `kubemetrics.STABLE`).\n```go\nvar someCounterVecMetric = kubemetrics.NewCounterVec(\n  \u0026prometheus.CounterOpts{\n    Name:           \"some_metric\",\n    Help:           \"some description\",\n    StabilityLevel: kubemetrics.STABLE,\n  },\n  []string{\"some-label\", \"other-label\"},\n}\n```\nPackage name `kubemetrics` can be replaced with local name of framework import.\n\nThose restrictions will allow AST based analysis to correctly interpret metric definitions.\n\n### Stable metric detection strategy\n\nStatic analysis will be done in two steps. First find stable metrics, second read and validate their fields.\n\nIn first one step we will want to distinguish stable metric without relying on their definition structure, allowing\nfreedom of declaration for non-stable metrics. This will be done by finding occurrences of metric options object initialization (`CounterOpts` for `Counter`).\nIf this initialization sets `StabilityLevel` then it will validate and read it.\nIf value could not be extracted then script should fail.\nMetrics which set `kubemetrics.STABLE` will be passed to second step.\n\nSecond step will be extracting information from stable metric definitions.\nThis step will be expecting exact structure of AST subtree of new metric call matching format proposed above.\nIf metric options object was initialized outside of new metric call or call structure deviates from expected format then\nstatic analysis should fail. This restriction will ensure that no field was read incorrectly or stable metric missed.\n\n### Format of stable metrics list file\n\nThis file will include all metric metadata that will serve source of documentation and a way to detect breaking change.\nEvery change to metric definition will require updating this file, thus a review from sig-instrumentation member.\nIt will be up to reviewer to decide which changes are backward compatible and which not.\n\nMetric information stored in file should be in easly readable and reviewable format.\nFor that `yaml` will be used (thanks for suggestion @ehashman). Example:\n```yaml\n- deprecatedVersion: \"1.16\"\n  help: \"some help information about the metric\"\n  labels:\n    - some-label\n  name: some_metric\n  namespace: some_namespace\n  stabilityLevel: STABLE\n  objectives:\n    0.5: 0.05\n    0.9: 0.01\n    0.99: 0.001\n  type: Summary\n```\n\nJson keys and lists will be sorted alphabetically to ensure that there is only one correct output.\n\n### Failure Modes\n\nIn cases where static analysis could not be done correctly (invalid stable metric format, etc.) script should\nfail (non zero response code) and write to STDERR information on problem. Example:\n```\npkg/controller/volume/persistentvolume/scheduler_bind_cache_metrics.go:42 stable metric CounterOpts should be invoked from newCounter\npkg/controller/volume/persistentvolume/scheduler_bind_cache_metrics.go:44 stable metric CounterOpts should be invoked from newCounter\nexit status 1\n```\n \n### Performance\n \nProposed static analysis takes around 4 seconds on one CPU core.\nThis time is comparable to conformance tests check ~2 sec and should be acceptable.\n\n## Graduation Criteria\n\n- [ ] Stable metric validation is enabled as pre-submit check\n\n## Implementation History\n\n- [ ] Implement static analysis script analysis all metrics types\n \n## Test Plan\n \n- [ ] Static analysis is covered by unit tests using common patterns of metric definition\n \n## Unresolved questions\n \n### Should code snippets used for unit testing be kept in separate files instead of strings?\n \nInput for static analysis unit tests will be Go code.\nIt could be beneficial to have them as separate files so they would be readable and interpretable by an IDE.\nProblem is that when put as separate files they would get BUILD file generated by gazelle and will no longer available\nas `data` for `go_test` rule. Possible ways to avoid that:\n* Change file extension. Using non standard file extension would lose benefit of separate files.\n* Excluding them in gazelle.\n"
+  },
+  {
+    "id": "445d96ea5624679096d6442cd8fe3e83",
+    "title": "Metrics Stability Framework to Beta",
+    "authors": ["@logicalhan", "@RainbowMango"],
+    "owningSig": "sig-instrumentation",
+    "participatingSigs": ["sig-instrumentation"],
+    "reviewers": ["@brancz"],
+    "approvers": ["@brancz"],
+    "editor": "@brancz",
+    "creationDate": "2019-10-28",
+    "lastUpdated": "2019-10-28",
+    "status": "implementable",
+    "seeAlso": [
+      "20181106-kubernetes-metrics-overhaul",
+      "20190404-kubernetes-control-plane-metrics-stability",
+      "20190605-metrics-stability-migration",
+      "20190605-metrics-validation-and-verification"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Metrics Stability Framework to Beta\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Remove Prometheus Registry](#remove-prometheus-registry)\n  - [Validated Import Restriction](#validated-import-restriction)\n  - [Deprecate Metrics](#deprecate-metrics)\n  - [Escape Hatch](#escape-hatch)\n- [Graduation Criteria](#graduation-criteria)\n- [Post-Beta tasks](#post-beta-tasks)\n- [Implementation History](#implementation-history)\n  - [Metrics Stability Framework](#metrics-stability-framework)\n  - [Metrics Stability Migration](#metrics-stability-migration)\n  - [Metrics Validation And Restriction](#metrics-validation-and-restriction)\n  - [Deprecate Metrics](#deprecate-metrics-1)\n  - [Escape Flag](#escape-flag)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThe metrics stability framework has been added to the Kubernetes project as a way to annotate metrics with a supported stability level. Depending on the stability level of a metric, there are some guarantees one can expect as a consumer (i.e. ingester) of a given metric. This document outline required steps to graduate it to Beta.\n\n## Motivation\n\nThe metrics stability framework is currently used for defining metrics stability levels for metrics in OSS Kubernetes. The motivation\nof this KEP is to address those feature requests and bug reports to move this feature to the Beta level.\n\n### Goals\n\nThese are the planned changes for Beta feature graduation:\n\n* No Kubernetes binaries register metrics to prometheus registries directly.\n* There is a validated import restriction on all kubernetes binaries (except `component-base/metrics`) such that we will fail, in a precommit phase, a direct import of prometheus in kubernetes. This forces all metrics related code to go through the metrics stability framework.\n* All currently deprecated metrics are deprecated using the `DeprecatedVersion` field of metrics options struct.\n* All Kubernetes binaries should have a command flag `--show-hidden-metrics` by which cluster admins can show metrics deprecated in last minor release.\n\n### Non-Goals\n\nThese are the issues considered and rejected for Beta:\n\n* Being able to individually turn off a metric (this will be a GA feature).\n\n## Proposal\n\n### Remove Prometheus Registry\nIn order to achieve the first goal: no binaries will register metrics to prometheus registries directly, we must have a plan for migrating metrics which are defined through the `prometheus.Collector` interface. These metrics currently do not have a way to express a stability level. @RainbowMango has a [PR with an implementation of how we may accomplish this](https://github.com/kubernetes/kubernetes/pull/83062/). Alternatively, we can just default all metrics which are defined through a custom `prometheus.Collector` as falling under stability level ALPHA, i.e. they do not offer stability guarantees. This buys us runway in bridging over to a solution like the one @RainbowMango proposes.\n\n### Validated Import Restriction\nWe also want to validate that direct prometheus imports are no longer possible in Kubernetes outside of component-base/metrics. This will force metric definition to occur within the stability framework and allow us to provide the guarantees that we intend. @serathius has some ideas in a [PR here](https://github.com/kubernetes/kubernetes/pull/84302).\n\n### Deprecate Metrics\nThe goal merely requires migrating over deprecated metrics from [PR](tdb).\n\n### Escape Hatch\nWe should add a command flag, such as `--show-hidden-metrics`, to each Kubernetes binaries.\nThis is to provide cluster admins an escape hatch to properly migrate off of a deprecated metric, if they were not able to react to the earlier deprecation warnings.\n\n\n## Graduation Criteria\n\nTo mark these as complete, all of the above features need to be implemented.\nAn [umbrella issue](https://github.com/kubernetes/kubernetes/issues/tdb) is tracking all of these changes.\nAlso there need to be sufficient tests for any of these new features and all existing features and documentation should be completed for all features.\n\nThere are still open questions that need to be addressed and updated in this KEP before graduation:\n\n## Post-Beta tasks\n\nThese are related Post-GA tasks:\n\n*\n\n## Implementation History\n\n### Metrics Stability Framework\n- Setup framework\n  - [x] https://github.com/kubernetes/kubernetes/pull/77037 (by @logicalhan)\n  - [x] https://github.com/kubernetes/kubernetes/pull/77618 (by @logicalhan)\n  - [x] https://github.com/kubernetes/kubernetes/pull/78773 (by @logicalhan)\n  - [x] https://github.com/kubernetes/kubernetes/pull/78867 (by @logicalhan)\n  - [x] https://github.com/kubernetes/kubernetes/pull/78877 (by @logicalhan)\n  - [x] https://github.com/kubernetes/kubernetes/pull/79237 (by @logicalhan)\n  - [x] https://github.com/kubernetes/kubernetes/pull/81190 (by @logicalhan)\n  - [x] https://github.com/kubernetes/kubernetes/pull/81395 (by @logicalhan)\n  - [x] https://github.com/kubernetes/kubernetes/pull/81579 (by @logicalhan)\n  - [x] https://github.com/kubernetes/kubernetes/pull/81608 (by @logicalhan)\n- Introduce bucket functionality\n  - [x] https://github.com/kubernetes/kubernetes/pull/82583 (by @RainbowMango)\n- Deal with stability default level\n  - [x] https://github.com/kubernetes/kubernetes/pull/82957 (by @RainbowMango)\n- Introduce label functionality\n  - [x] https://github.com/kubernetes/kubernetes/pull/83019 (by @RainbowMango)\n- Introduce test util:\n  - [x] https://github.com/kubernetes/kubernetes/pull/83299 (by @RainbowMango)\n  - [x] https://github.com/kubernetes/kubernetes/pull/83699 (by @RainbowMango)\n- Introduce http handler functionality\n  - [x] https://github.com/kubernetes/kubernetes/pull/83722 (by @RainbowMango)\n- Introduce GaugeFunc\n  - [X] https://github.com/kubernetes/kubernetes/pull/83830 (by @RainbowMango)\n- Introduce custom collector\n  - [ ] https://github.com/kubernetes/kubernetes/pull/83062 (by @RainbowMango)\n- Cleanup\n  - [ ] https://github.com/kubernetes/kubernetes/pull/84135 (by @RainbowMango)\n  - [x] https://github.com/kubernetes/kubernetes/pull/81432 (by @logicalhan)\n- Bug fix\n  - [x] https://github.com/kubernetes/kubernetes/pull/84395 (by @RainbowMango)\n\n### Metrics Stability Migration\n- General Migration\n  - [x] for shared metrics: https://github.com/kubernetes/kubernetes/pull/81173 (by @logicalhan)\n  - [x] for apiserver: https://github.com/kubernetes/kubernetes/pull/81531 (by @logicalhan)\n  - [x] for kubelet: https://github.com/kubernetes/kubernetes/pull/81534 (by @logicalhan)\n  - [x] for scheduler: https://github.com/kubernetes/kubernetes/pull/81576 (by @logicalhan)\n  - [x] for controller manager: https://github.com/kubernetes/kubernetes/pull/81624 (by @logicalhan)\n  - [x] for kube-proxy: https://github.com/kubernetes/kubernetes/pull/81626 (by @logicalhan)\n  - [x] for etcd version monitor: https://github.com/kubernetes/kubernetes/pull/83283 (by @RainbowMango)\n  - [ ] for metrics validation framework: https://github.com/kubernetes/kubernetes/pull/84500 (by @RainbowMango)\n- Migrate bucket functionality\n  - [x] https://github.com/kubernetes/kubernetes/pull/82626 (by @RainbowMango)\n  - [x] https://github.com/kubernetes/kubernetes/pull/82630 (by @RainbowMango)\n  - [x] https://github.com/kubernetes/kubernetes/pull/82736 (by @RainbowMango)\n  - [x] https://github.com/kubernetes/kubernetes/pull/82737 (by @RainbowMango)\n  - [x] https://github.com/kubernetes/kubernetes/pull/82741 (by @RainbowMango)\n  - [x] https://github.com/kubernetes/kubernetes/pull/82745 (by @RainbowMango)\n- Migrate bucket functionality\n  - [x] https://github.com/kubernetes/kubernetes/pull/83159 (by @RainbowMango)\n  - [x] https://github.com/kubernetes/kubernetes/pull/83220 (by @RainbowMango)\n  - [x] https://github.com/kubernetes/kubernetes/pull/83223 (by @RainbowMango)\n  - [x] https://github.com/kubernetes/kubernetes/pull/83269 (by @RainbowMango)\n  - [x] https://github.com/kubernetes/kubernetes/pull/83278 (by @RainbowMango)\n  - [x] https://github.com/kubernetes/kubernetes/pull/83279 (by @RainbowMango)\n- Migrate or refactor test case\n  - [x] https://github.com/kubernetes/kubernetes/pull/83611 (by @RainbowMango)\n  - [x] https://github.com/kubernetes/kubernetes/pull/83678 (by @RainbowMango)\n  - [x] https://github.com/kubernetes/kubernetes/pull/83713 (by @RainbowMango)\n  - [ ] https://github.com/kubernetes/kubernetes/pull/83664 (by @RainbowMango)\n  - [x] https://github.com/kubernetes/kubernetes/pull/84283 (by @serathius)\n- Migrate promhttp\n  - [ ] https://github.com/kubernetes/kubernetes/pull/84393 (by @wuyafang)\n  - [x] https://github.com/kubernetes/kubernetes/pull/84221 (by @wuyafang)\n\n### Metrics Validation And Restriction\n- [x] https://github.com/kubernetes/kubernetes/pull/80803 (by @serathius)\n- [x] https://github.com/kubernetes/kubernetes/pull/80906 (by @serathius)\n- [x] https://github.com/kubernetes/kubernetes/pull/81510 (by @serathius)\n- [ ] https://github.com/kubernetes/kubernetes/pull/84302 (by @serathius)\n- [ ] https://github.com/kubernetes/kubernetes/pull/84373 (by @serathius)\n- [ ] https://github.com/kubernetes/kubernetes/pull/84378 (by @serathius)\n\n### Deprecate Metrics\n- [ ] https://github.com/kubernetes/kubernetes/pull/83836 (by @RainbowMango)\n- [ ] https://github.com/kubernetes/kubernetes/pull/83837 (by @RainbowMango)\n- [ ] https://github.com/kubernetes/kubernetes/pull/83838 (by @RainbowMango)\n- [ ] https://github.com/kubernetes/kubernetes/pull/83839 (by @RainbowMango)\n- [ ] https://github.com/kubernetes/kubernetes/pull/83841 (by @RainbowMango)\n\n### Escape Flag\n- [ ] https://github.com/kubernetes/kubernetes/pull/84292 (by @RainbowMango)\n"
+  },
+  {
+    "id": "cc7025f8253d5301697c6f94f9316380",
+    "title": "Pod Ready++",
+    "authors": ["freehan@"],
+    "owningSig": "sig-network",
+    "participatingSigs": ["sig-node", "sig-cli"],
+    "reviewers": ["thockin@", "dchen1107@"],
+    "approvers": ["thockin@", "dchen1107@"],
+    "editor": "freehan@",
+    "creationDate": "2018-04-01",
+    "lastUpdated": "2018-04-01",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Pod Ready++\n\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [PodSpec](#podspec)\n    - [Constraints:](#constraints)\n  - [Pod Readiness](#pod-readiness)\n  - [Custom Pod Condition](#custom-pod-condition)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n      - [Workloads](#workloads)\n      - [Kubelet](#kubelet)\n  - [Feature Integration](#feature-integration)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n      - [Why not fix the workloads?](#why-not-fix-the-workloads)\n      - [Why not extend container readiness?](#why-not-extend-container-readiness)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis proposal aims to add extensibility to pod readiness. Besides container readiness, external feedback can be injected into PodStatus and influence pod readiness. Thus, achieving pod “ready++”. \n\n## Motivation\n\nPod readiness indicates whether the pod is ready to serve traffic. Pod readiness is dictated by kubelet with user specified readiness probe. On the other hand, pod readiness determines whether pod address shows up on the address list on related endpoints object. K8s primitives that manage pods, such as Deployment, only takes pod status into account for decision making, such as advancement during rolling update. \n\nFor example, during deployment rolling update, a new pod becomes ready. On the other hand, service, network policy and load-balancer are not yet ready for the new pod due to whatever reason (e.g. slowness in api machinery, endpoints controller, kube-proxy, iptables or infrastructure programming). This may cause service disruption or lost of backend capacity. In extreme cases, if rolling update completes before any new replacement pod actually start serving traffic, this will cause service outage. \n\n\n### Goals\n\n- Allow extra signals for pod readiness.\n\n### Non-Goals\n\n- Provide generic framework to solve all transition problems in k8s (e.g. blue green deployment).\n\n## Proposal\n\n[K8s Proposal: Pod Ready++](https://docs.google.com/document/d/1VFZbc_IqPf_Msd-jul7LKTmGjvQ5qRldYOFV0lGqxf8/edit#)\n\n### PodSpec\nIntroduce an extra field called ReadinessGates in PodSpec. The field stores a list of ReadinessGate structure as follows: \n```yaml\ntype ReadinessGate struct {\n\tconditionType string\t\n}\n```\nThe ReadinessGate struct has only one string field called ConditionType. ConditionType refers to a condition in the PodCondition list in PodStatus. And the status of conditions specified in the ReadinessGates will be evaluated for pod readiness. If the condition does not exist in the PodCondition list, its status will be default to false. \n\n#### Constraints:\n- ReadinessGates can only be specified at pod creation. \n- No Update allowed on ReadinessGates.\n- ConditionType must conform to the naming convention of custom pod condition.\n\n### Pod Readiness\nChange the pod readiness definition to as follows:\n```\nPod is ready == containers are ready AND conditions in ReadinessGates are True\n```\nKubelet will evaluate conditions specified in ReadinessGates and update the pod “Ready” status. For example, in the following pod spec, two readinessGates are specified. The status of “www.example.com/feature-1” is false, hence the pod is not ready. \n\n```yaml\nKind: Pod \n… \nspec: \n  readinessGates:\n  - conditionType: www.example.com/feature-1\n  - conditionType: www.example.com/feature-2\n… \nstatus:\n  conditions:\n  - lastProbeTime: null\n    lastTransitionTime: 2018-01-01T00:00:00Z\n    status: \"False\"\n    type: Ready\n  - lastProbeTime: null\n    lastTransitionTime: 2018-01-01T00:00:00Z\n    status: \"False\"\n    type: www.example.com/feature-1\n  - lastProbeTime: null\n    lastTransitionTime: 2018-01-01T00:00:00Z\n    status: \"True\"\n    type: www.example.com/feature-2\n  containerStatuses:\n  - containerID: docker://xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n    ready : true\n… \n```\n\nAnother pod condition `ContainerReady` will be introduced to capture the old pod `Ready` condition. \n```\nContainerReady is true == containers are ready\n```\n\n### Custom Pod Condition\nCustom pod condition can be injected thru PATCH action using KubeClient. Please be noted that “kubectl patch” does not support patching object status. Need to use client-go or other KubeClient implementations. \n\nNaming Convention:\nThe type of custom pod condition must comply with k8s label key format. For example, “www.example.com/feature-1”.\n\n\n### Implementation Details/Notes/Constraints\n\n##### Workloads\nTo conform with this proposals, workload controllers MUST take pod “Ready” condition as the final signal to proceed during transitions.  \n\nFor the workloads that take pod readiness as a critical signal for its decision making, they will automatically comply with this proposal without any change. Majority, if not all, of the workloads satisfy this condition. \n\n##### Kubelet\n- Use PATCH instead of PUT to update PodStatus fields that are dictated by kubelet. \n- Only compare the fields that managed by kubelet for PodStatus reconciliation .\n- Watch PodStatus changes and evaluate ReadinessGates for pod readiness.\n\n### Feature Integration\nIn this section, we will discuss how to make ReadinessGates transparent to K8s API user. In order words, a K8s API user does not need to specify ReadinessGates to use specific features. This allows existing manifests to just work with features that require ReadinessGate.\nEach feature will bear the burden of injecting ReadinessGate and keep its custom pod condition in sync. ReadinessGate can be injected using mutating webhook at pod creation time. After pod creation, each feature is responsible for keeping its custom pod condition in sync as long as its ReadinessGate exists in the PodSpec. This can be achieved by running k8s controller to sync conditions on relevant pods. This is to ensure that PodStatus is observable and recoverable even when catastrophic failure (e.g. loss of data) occurs at API server. \n\n\n\n### Risks and Mitigations\n\nRisks:\n- Features that utilize the extension point from this proposal may abuse the API.  \n- User confusion on pod ready++\n\nMitigations:\n- Better specification and API validation.\n- Better CLI/UI/UX\n  \n\n## Graduation Criteria\n\n- Kubelet changes should not have any impact on kubelet reliability. \n- Feature integration with the pod ready++ extension.\n\n\n## Implementation History\n\nTBD\n\n\n## Alternatives \n\n##### Why not fix the workloads?\n\nThere are a lot of workloads including core workloads such as deployment and 3rd party workloads such as spark operator. Most if not all of them take pod readiness as a critical signal for decision making, while ignoring higher level abstractions (e.g. service, network policy and ingress). To complicate the problem more, label selector makes membership relationship implicit and dynamic. Solving this problem in all workload controllers would require much bigger change than this proposal. \n\n##### Why not extend container readiness?\n\nContainer readiness is tied to low level constructs such as runtime. This inherently implies that the kubelet and underlying system has full knowledge of container status. Injecting external feedback into container status would complicate the abstraction and control flow. Meanwhile, higher level abstractions (e.g. service) generally takes pod as the atom instead of container. \n"
+  },
+  {
+    "id": "2cd9385e529545be32cc2e1cc1421275",
+    "title": "Graduate CoreDNS to GA",
+    "authors": ["@johnbelamaric", "@rajansandeep"],
+    "owningSig": "sig-network",
+    "participatingSigs": ["sig-cluster-lifecycle"],
+    "reviewers": ["@bowei", "@thockin"],
+    "approvers": ["@thockin"],
+    "editor": "@rajansandeep",
+    "creationDate": "2018-03-21",
+    "lastUpdated": "2018-05-18",
+    "status": "provisional",
+    "seeAlso": ["https://github.com/kubernetes/community/pull/2167"],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Graduate CoreDNS to GA\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Use Cases](#use-cases)\n    - [Configuring CoreDNS](#configuring-coredns)\n    - [Kubeadm](#kubeadm)\n    - [Kube-up](#kube-up)\n    - [Minikube](#minikube)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nCoreDNS is sister CNCF project and is the successor to SkyDNS, on which kube-dns is based. It is a flexible, extensible\nauthoritative DNS server and directly integrates with the Kubernetes API. It can serve as cluster DNS,\ncomplying with the [dns spec](https://git.k8s.io/dns/docs/specification.md). As an independent project,\nit is more actively developed than kube-dns and offers performance and functionality beyond what kube-dns has. For more details, see the [introductory presentation](https://docs.google.com/presentation/d/1v6Coq1JRlqZ8rQ6bv0Tg0usSictmnN9U80g8WKxiOjQ/edit#slide=id.g249092e088_0_181), or [coredns.io](https://coredns.io), or the [CNCF webinar](https://youtu.be/dz9S7R8r5gw).\n\nCurrently, we are following the road-map defined [here](https://github.com/kubernetes/features/issues/427). CoreDNS is Beta in Kubernetes v1.10, which can be installed as an alternate to kube-dns.\nThe purpose of this proposal is to graduate CoreDNS to GA.\n\n## Motivation\n\n* CoreDNS is more flexible and extensible than kube-dns. \n* CoreDNS is easily extensible and maintainable using a plugin architecture.\n* CoreDNS has fewer moving parts than kube-dns, taking advantage of the plugin architecture, making it a single executable and single process.\n* It is written in Go, making it memory-safe (kube-dns includes dnsmasq which is not). \n* CoreDNS has [better performance](https://github.com/kubernetes/community/pull/1100#issuecomment-337747482) than [kube-dns](https://github.com/kubernetes/community/pull/1100#issuecomment-338329100) in terms of greater QPS, lower latency, and lower memory consumption. \n\n### Goals\n\n* Bump up CoreDNS to be GA.\n* Make CoreDNS available as an image in a Kubernetes repository (To Be Defined) and ensure a workflow/process to update the CoreDNS versions in the future.\n  May be deferred to [next KEP](https://github.com/kubernetes/community/pull/2167) if goal not achieved in time.\n* Provide a kube-dns to CoreDNS upgrade path with configuration translation in `kubeadm`.\n* Provide a CoreDNS to CoreDNS upgrade path in `kubeadm`.\n\n### Non-Goals\n\n* Translation of CoreDNS ConfigMap back to kube-dns (i.e., downgrade).\n* Translation configuration of kube-dns to equivalent CoreDNS that is defined outside of the kube-dns ConfigMap. For example, modifications to the manifest or `dnsmasq` configuration.\n* Fate of kube-dns in future releases, i.e. deprecation path.\n* Making [CoreDNS the default](https://github.com/kubernetes/community/pull/2167) in every installer.\n\n## Proposal\n\nThe proposed solution is to enable the selection of CoreDNS as a GA cluster service discovery DNS for Kubernetes.\nSome of the most used deployment tools have been upgraded by the CoreDNS team, in cooperation of the owners of these tools, to be able to deploy CoreDNS:\n* kubeadm\n* kube-up\n* minikube\n* kops\n\nFor other tools, each maintainer would have to add the upgrade to CoreDNS.\n\n### Use Cases\n\n* CoreDNS supports all functionality of kube-dns and also addresses [several use-cases kube-dns lacks](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/network/coredns.md#use-cases). Some of the Use Cases are as follows: \n    * Supporting [Autopath](https://coredns.io/plugins/autopath/), which reduces the high query load caused by the long DNS search path in Kubernetes.\n    * Making an alias for an external name [#39792](https://github.com/kubernetes/kubernetes/issues/39792)\n    \n* By default, the user experience would be unchanged. For more advanced uses, existing users would need to modify the ConfigMap that contains the CoreDNS configuration file.\n* Since CoreDNS has more supporting features than kube-dns, there will be no path to retain the CoreDNS configuration in case a user wants to switch to kube-dns.\n\n#### Configuring CoreDNS\n\nThe CoreDNS configuration file is called a `Corefile` and syntactically is the same as a [Caddyfile](https://caddyserver.com/docs/caddyfile). The file consists of multiple stanzas called _server blocks_.\nEach of these represents a set of zones for which that server block should respond, along with the list of plugins to apply to a given request. More details on this can be found in the \n[Corefile Explained](https://coredns.io/2017/07/23/corefile-explained/) and [How Queries Are Processed](https://coredns.io/2017/06/08/how-queries-are-processed-in-coredns/) blog entries.\n\nThe following can be expected when CoreDNS is graduated to GA.\n\n#### Kubeadm\n\n* The CoreDNS feature-gates flag will be marked as GA.\n* As Kubeadm maintainers chose to deploy CoreDNS as the default Cluster DNS for Kubernetes 1.11:\n    * CoreDNS will be installed by default in a fresh install of Kubernetes via kubeadm.\n    * For users upgrading Kubernetes via kubeadm, it will install CoreDNS by default whether the user had kube-dns or CoreDNS in a previous kubernetes version.\n    * In case a user wants to install kube-dns instead of CoreDNS, they have to set the feature-gate of CoreDNS to false. `--feature-gates=CoreDNS=false`\n* When choosing to install CoreDNS, the configmap of a previously installed kube-dns will be automatically translated to the equivalent CoreDNS configmap.\n\n#### Kube-up\n\n* CoreDNS will be installed when the environment variable `CLUSTER_DNS_CORE_DNS` is set to `true`. The default value is `false`.\n\n#### Minikube\n\n* CoreDNS to be an option in the add-on manager, with CoreDNS disabled by default.\n\n## Graduation Criteria\n\n* Verify that all e2e conformance and DNS related tests (xxx-kubernetes-e2e-gce, ci-kubernetes-e2e-gce-gci-ci-master and filtered by `--ginkgo.skip=\\\\[Slow\\\\]|\\\\[Serial\\\\]|\\\\[Disruptive\\\\]|\\\\[Flaky\\\\]|\\\\[Feature:.+\\\\]`) run successfully for CoreDNS.\n  None of the tests successful with Kube-DNS should be failing with CoreDNS.\n* Add CoreDNS as part of the e2e Kubernetes scale runs and ensure tests are not failing.\n* Extend [perf-tests](https://github.com/kubernetes/perf-tests/tree/master/dns) for CoreDNS.\n* Add a dedicated DNS related tests in e2e scalability test [Feature:performance].\n\n## Implementation History\n\n* 20170912 - [Feature proposal](https://github.com/kubernetes/features/issues/427) for CoreDNS to be implemented as the default DNS in Kubernetes.\n* 20171108 - Successfully released [CoreDNS as an Alpha feature-gate in Kubernetes v1.9](https://github.com/kubernetes/kubernetes/pull/52501).\n* 20180226 - CoreDNS graduation to Incubation in CNCF.\n* 20180305 - Support for Kube-dns configmap translation and move up [CoreDNS to Beta](https://github.com/kubernetes/kubernetes/pull/58828) for Kubernetes v1.10.\n"
+  },
+  {
+    "id": "e5cd170a313f2ebd4fb538d7ae43d2ec",
+    "title": "IPVS Load Balancing Mode in Kubernetes",
+    "authors": ["@rramkumar1"],
+    "owningSig": "sig-network",
+    "participatingSigs": null,
+    "reviewers": ["@thockin", "@m1093782566"],
+    "approvers": ["@thockin", "@m1093782566"],
+    "editor": "@thockin",
+    "creationDate": "2018-03-21",
+    "lastUpdated": "2019-06-27",
+    "status": "implemented",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# IPVS Load Balancing Mode in Kubernetes\n\n**Note: This is a retroactive KEP. Credit goes to @m1093782566, @haibinxie, and @quinton-hoole for all information \u0026 design in this KEP.**\n\n**Important References: https://github.com/kubernetes/community/pull/692/files**\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-goals](#non-goals)\n  - [Challenges and Open Questions [optional]](#challenges-and-open-questions-optional)\n- [Proposal](#proposal)\n  - [Kube-Proxy Parameter Changes](#kube-proxy-parameter-changes)\n  - [Build Changes](#build-changes)\n  - [Deployment Changes](#deployment-changes)\n  - [Design Considerations](#design-considerations)\n    - [IPVS service network topology](#ipvs-service-network-topology)\n    - [Port remapping](#port-remapping)\n    - [Falling back to iptables](#falling-back-to-iptables)\n    - [Supporting NodePort service](#supporting-nodeport-service)\n    - [Supporting ClusterIP service](#supporting-clusterip-service)\n  - [Support LoadBalancer service](#support-loadbalancer-service)\n  - [Support Only NodeLocal Endpoints](#support-only-nodelocal-endpoints)\n    - [Session affinity](#session-affinity)\n    - [Cleaning up inactive rules](#cleaning-up-inactive-rules)\n    - [Sync loop pseudo code](#sync-loop-pseudo-code)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Drawbacks [optional]](#drawbacks-optional)\n- [Alternatives [optional]](#alternatives-optional)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nWe are building a new implementation of kube proxy built on top of IPVS (IP Virtual Server).\n\n## Motivation\n\nAs Kubernetes grows in usage, the scalability of its resources becomes more and more\nimportant. In particular, the scalability of services is paramount to the adoption of Kubernetes\nby developers/companies running large workloads. Kube Proxy, the building block of service routing\nhas relied on the battle-hardened iptables to implement the core supported service types such as\nClusterIP and NodePort. However, iptables struggles to scale to tens of thousands of services because\nit is designed purely for firewalling purposes and is based on in-kernel rule chains. On the\nother hand, IPVS is specifically designed for load balancing and uses more efficient data structures\nunder the hood. For more information on the performance benefits of IPVS vs. iptables, take a look\nat these [slides](https://docs.google.com/presentation/d/1BaIAywY2qqeHtyGZtlyAp89JIZs59MZLKcFLxKE6LyM/edit?usp=sharing).\n\n### Goals\n\n* Improve the performance of services\n\n### Non-goals\n\nNone\n\n### Challenges and Open Questions [optional]\n\nNone\n\n\n## Proposal\n\n### Kube-Proxy Parameter Changes\n\n***Parameter: --proxy-mode***\nIn addition to existing userspace and iptables modes, IPVS mode is configured via --proxy-mode=ipvs. In the initial implementation, it implicitly uses IPVS [NAT](http://www.linuxvirtualserver.org/VS-NAT.html) mode.\n\n***Parameter: --ipvs-scheduler***\nA new kube-proxy parameter will be added to specify the IPVS load balancing algorithm, with the parameter being --ipvs-scheduler. If it’s not configured, then round-robin (rr) is default value. If it’s incorrectly configured, then kube-proxy will exit with error message.\n  * rr: round-robin\n  * lc: least connection\n  * dh: destination hashing\n  * sh: source hashing\n  * sed: shortest expected delay\n  * nq: never queue\nFor more details, refer to http://kb.linuxvirtualserver.org/wiki/Ipvsadm\n\nIn future, we can implement service specific scheduler (potentially via annotation), which has higher priority and overwrites the value.\n\n***Parameter: --cleanup-ipvs***\nSimilar to the --cleanup-iptables parameter, if true, cleanup IPVS configuration and IPTables rules that are created in IPVS mode.\n\n***Parameter: --ipvs-sync-period***\nMaximum interval of how often IPVS rules are refreshed (e.g. '5s', '1m'). Must be greater than 0.\n\n***Parameter: --ipvs-min-sync-period***\nMinimum interval of how often the IPVS rules are refreshed (e.g. '5s', '1m'). Must be greater than 0.\n\n\n### Build Changes\n\nNo changes at all. The IPVS implementation is built on [docker/libnetwork](https://godoc.org/github.com/docker/libnetwork/ipvs) IPVS library, which is a pure-golang implementation and talks to kernel via socket communication.\n\n### Deployment Changes\n\nIPVS kernel module installation is beyond Kubernetes. It’s assumed that IPVS kernel modules are installed on the node before running kube-proxy. When kube-proxy starts, if the proxy mode is IPVS, kube-proxy would validate if IPVS modules are installed on the node. If it’s not installed, then kube-proxy will fall back to the iptables proxy mode.\n\n### Design Considerations\n\n#### IPVS service network topology\n\nWe will create a dummy interface and assign all kubernetes service ClusterIP's to the dummy interface (default name is `kube-ipvs0`). For example,\n\n```shell\n# ip link add kube-ipvs0 type dummy\n# ip addr\n...\n73: kube-ipvs0: \u003cBROADCAST,NOARP\u003e mtu 1500 qdisc noop state DOWN qlen 1000\n    link/ether 26:1f:cc:f8:cd:0f brd ff:ff:ff:ff:ff:ff\n\n#### Assume 10.102.128.4 is service Cluster IP\n# ip addr add 10.102.128.4/32 dev kube-ipvs0\n...\n73: kube-ipvs0: \u003cBROADCAST,NOARP\u003e mtu 1500 qdisc noop state DOWN qlen 1000\n    link/ether 1a:ce:f5:5f:c1:4d brd ff:ff:ff:ff:ff:ff\n    inet 10.102.128.4/32 scope global kube-ipvs0\n       valid_lft forever preferred_lft forever\n```\n\nNote that the relationship between a Kubernetes service and an IPVS service is `1:N`. Consider a Kubernetes service that has more than one access IP. For example, an External IP type service has 2 access IP's (ClusterIP and External IP). Then the IPVS proxier will create 2 IPVS services - one for Cluster IP and the other one for External IP.\n\nThe relationship between a Kubernetes endpoint and an IPVS destination is `1:1`.\nFor instance, deletion of a Kubernetes service will trigger deletion of the corresponding IPVS service and address bound to dummy interface.\n\n\n#### Port remapping\n\nThere are 3 proxy modes in ipvs - NAT (masq), IPIP and DR. Only NAT mode supports port remapping. We will use IPVS NAT mode in order to supporting port remapping. The following example shows ipvs mapping service port `3080` to container port `8080`.\n\n```shell\n# ipvsadm -ln\nIP Virtual Server version 1.2.1 (size=4096)\nPort LocalAddress:Port Scheduler Flags\n  -\u003e RemoteAddress:Port           Forward Weight ActiveConn InActConn     \nTCP  10.102.128.4:3080 rr\n  -\u003e 10.244.0.235:8080            Masq    1      0          0         \n  -\u003e 10.244.1.237:8080            Masq    1      0          0     \n\n```\n\n#### Falling back to iptables\n\nIPVS proxier will employ iptables in doing packet filtering, SNAT and supporting NodePort type service. Specifically, ipvs proxier will fall back on iptables in the following 4 scenarios.\n\n* kube-proxy start with --masquerade-all=true\n* Specify cluster CIDR in kube-proxy startup\n* Load Balancer Source Ranges is specified for LB type service\n* Support NodePort type service\n\nAnd, IPVS proxier will maintain 5 kubernetes-specific chains in nat table\n\n- KUBE-POSTROUTING\n- KUBE-MARK-MASQ\n- KUBE-MARK-DROP\n\n`KUBE-POSTROUTING`, `KUBE-MARK-MASQ`, ` KUBE-MARK-DROP` are maintained by kubelet and ipvs proxier won't create them. IPVS proxier will make sure chains `KUBE-MARK-SERVICES` and `KUBE-NODEPORTS` exist in its sync loop.\n\n**1. kube-proxy start with --masquerade-all=true**\n\nIf kube-proxy starts with `--masquerade-all=true`, the IPVS proxier will masquerade all traffic accessing service ClusterIP, which behaves same as what iptables proxier does.\nSuppose there is a service with Cluster IP `10.244.5.1` and port `8080`:\n\n```shell\n# iptables -t nat -nL\n\nChain PREROUTING (policy ACCEPT)\ntarget     prot opt source               destination         \nKUBE-SERVICES  all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes service portals */\n\nChain OUTPUT (policy ACCEPT)\ntarget     prot opt source               destination         \nKUBE-SERVICES  all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes service portals */\n\nChain POSTROUTING (policy ACCEPT)\ntarget     prot opt source               destination         \nKUBE-POSTROUTING  all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes postrouting rules */\n\nChain KUBE-POSTROUTING (1 references)\ntarget     prot opt source               destination         \nMASQUERADE  all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes service traffic requiring SNAT */ mark match 0x4000/0x4000\n\nChain KUBE-MARK-DROP (0 references)\ntarget     prot opt source               destination         \nMARK       all  --  0.0.0.0/0            0.0.0.0/0            MARK or 0x8000\n\nChain KUBE-MARK-MASQ (6 references)\ntarget     prot opt source               destination         \nMARK       all  --  0.0.0.0/0            0.0.0.0/0            MARK or 0x4000\n\nChain KUBE-SERVICES (2 references)\ntarget     prot opt source               destination         \nKUBE-MARK-MASQ  tcp  -- 0.0.0.0/0        10.244.5.1            /* default/foo:http cluster IP */ tcp dpt:8080\n```\n\n**2. Specify cluster CIDR in kube-proxy startup**\n\nIf kube-proxy starts with `--cluster-cidr=\u003ccidr\u003e`, the IPVS proxier will masquerade off-cluster traffic accessing service ClusterIP, which behaves same as what iptables proxier does.\nSuppose kube-proxy is provided with the cluster cidr `10.244.16.0/24`, and service Cluster IP is `10.244.5.1` and port is `8080`:\n\n```shell\n# iptables -t nat -nL\n\nChain PREROUTING (policy ACCEPT)\ntarget     prot opt source               destination         \nKUBE-SERVICES  all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes service portals */\n\nChain OUTPUT (policy ACCEPT)\ntarget     prot opt source               destination         \nKUBE-SERVICES  all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes service portals */\n\nChain POSTROUTING (policy ACCEPT)\ntarget     prot opt source               destination         \nKUBE-POSTROUTING  all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes postrouting rules */\n\nChain KUBE-POSTROUTING (1 references)\ntarget     prot opt source               destination         \nMASQUERADE  all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes service traffic requiring SNAT */ mark match 0x4000/0x4000\n\nChain KUBE-MARK-DROP (0 references)\ntarget     prot opt source               destination         \nMARK       all  --  0.0.0.0/0            0.0.0.0/0            MARK or 0x8000\n\nChain KUBE-MARK-MASQ (6 references)\ntarget     prot opt source               destination         \nMARK       all  --  0.0.0.0/0            0.0.0.0/0            MARK or 0x4000\n\nChain KUBE-SERVICES (2 references)\ntarget     prot opt source               destination         \nKUBE-MARK-MASQ  tcp  -- !10.244.16.0/24        10.244.5.1            /* default/foo:http cluster IP */ tcp dpt:8080\n```\n\n**3. Load Balancer Source Ranges is specified for LB type service**\n\nWhen service's `LoadBalancerStatus.ingress.IP` is not empty and service's `LoadBalancerSourceRanges` is specified, IPVS proxier will install iptables rules which looks like what is shown below.\n\nSuppose service's `LoadBalancerStatus.ingress.IP` is `10.96.1.2` and service's `LoadBalancerSourceRanges` is `10.120.2.0/24`:\n\n```shell\n# iptables -t nat -nL\n\nChain PREROUTING (policy ACCEPT)\ntarget     prot opt source               destination         \nKUBE-SERVICES  all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes service portals */\n\nChain OUTPUT (policy ACCEPT)\ntarget     prot opt source               destination         \nKUBE-SERVICES  all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes service portals */\n\nChain POSTROUTING (policy ACCEPT)\ntarget     prot opt source               destination         \nKUBE-POSTROUTING  all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes postrouting rules */\n\nChain KUBE-POSTROUTING (1 references)\ntarget     prot opt source               destination         \nMASQUERADE  all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes service traffic requiring SNAT */ mark match 0x4000/0x4000\n\nChain KUBE-MARK-DROP (0 references)\ntarget     prot opt source               destination         \nMARK       all  --  0.0.0.0/0            0.0.0.0/0            MARK or 0x8000\n\nChain KUBE-MARK-MASQ (6 references)\ntarget     prot opt source               destination         \nMARK       all  --  0.0.0.0/0            0.0.0.0/0            MARK or 0x4000\n\nChain KUBE-SERVICES (2 references)\ntarget     prot opt source       destination         \nACCEPT  tcp  -- 10.120.2.0/24    10.96.1.2       /* default/foo:http loadbalancer IP */ tcp dpt:8080\nDROP    tcp  -- 0.0.0.0/0        10.96.1.2       /* default/foo:http loadbalancer IP */ tcp dpt:8080\n```\n\n**4. Support NodePort type service**\n\nPlease check the section below.\n\n#### Supporting NodePort service\n\nFor supporting NodePort type service, iptables will recruit the existing implementation in the iptables proxier. For example,\n\n```shell\n# kubectl describe svc nginx-service\nName:\t\t\tnginx-service\n...\nType:\t\t\tNodePort\nIP:\t\t\t    10.101.28.148\nPort:\t\t\thttp\t3080/TCP\nNodePort:\t\thttp\t31604/TCP\nEndpoints:\t\t172.17.0.2:80\nSession Affinity:\tNone\n\n# iptables -t nat -nL\n\n[root@100-106-179-225 ~]# iptables -t nat -nL\nChain PREROUTING (policy ACCEPT)\ntarget     prot opt source               destination         \nKUBE-SERVICES  all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes service portals */\n\nChain OUTPUT (policy ACCEPT)\ntarget     prot opt source               destination         \nKUBE-SERVICES  all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes service portals */\n\nChain KUBE-SERVICES (2 references)\ntarget     prot opt source               destination         \nKUBE-MARK-MASQ  tcp  -- !172.16.0.0/16        10.101.28.148        /* default/nginx-service:http cluster IP */ tcp dpt:3080\nKUBE-SVC-6IM33IEVEEV7U3GP  tcp  --  0.0.0.0/0            10.101.28.148        /* default/nginx-service:http cluster IP */ tcp dpt:3080\nKUBE-NODEPORTS  all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes service nodeports; NOTE: this must be the last rule in this chain */ ADDRTYPE match dst-type LOCAL\n\nChain KUBE-NODEPORTS (1 references)\ntarget     prot opt source               destination         \nKUBE-MARK-MASQ  tcp  --  0.0.0.0/0            0.0.0.0/0            /* default/nginx-service:http */ tcp dpt:31604\nKUBE-SVC-6IM33IEVEEV7U3GP  tcp  --  0.0.0.0/0            0.0.0.0/0            /* default/nginx-service:http */ tcp dpt:31604\n\nChain KUBE-SVC-6IM33IEVEEV7U3GP (2 references)\ntarget     prot opt source               destination\nKUBE-SEP-Q3UCPZ54E6Q2R4UT  all  --  0.0.0.0/0            0.0.0.0/0            /* default/nginx-service:http */\nChain KUBE-SEP-Q3UCPZ54E6Q2R4UT (1 references)\ntarget     prot opt source               destination         \nKUBE-MARK-MASQ  all  --  172.17.0.2           0.0.0.0/0            /* default/nginx-service:http */\nDNAT  \n```\n\n#### Supporting ClusterIP service\n\nWhen creating a ClusterIP type service, IPVS proxier will do 3 things:\n\n* make sure dummy interface exists in the node\n* bind service cluster IP to the dummy interface\n* create an IPVS service whose address corresponds to the Kubernetes service Cluster IP.\n\nFor example,\n\n```shell\n# kubectl describe svc nginx-service\nName:\t\t\tnginx-service\n...\nType:\t\t\tClusterIP\nIP:\t\t\t    10.102.128.4\nPort:\t\t\thttp\t3080/TCP\nEndpoints:\t\t10.244.0.235:8080,10.244.1.237:8080\nSession Affinity:\tNone\n\n# ip addr\n...\n73: kube-ipvs0: \u003cBROADCAST,NOARP\u003e mtu 1500 qdisc noop state DOWN qlen 1000\n    link/ether 1a:ce:f5:5f:c1:4d brd ff:ff:ff:ff:ff:ff\n    inet 10.102.128.4/32 scope global kube-ipvs0\n       valid_lft forever preferred_lft forever\n\n# ipvsadm -ln\nIP Virtual Server version 1.2.1 (size=4096)\nProt LocalAddress:Port Scheduler Flags\n  -\u003e RemoteAddress:Port           Forward Weight ActiveConn InActConn     \nTCP  10.102.128.4:3080 rr\n  -\u003e 10.244.0.235:8080            Masq    1      0          0         \n  -\u003e 10.244.1.237:8080            Masq    1      0          0   \n```\n\n### Support LoadBalancer service\n\nIPVS proxier will NOT bind LB's ingress IP to the dummy interface. When creating a LoadBalancer type service, ipvs proxier will do 4 things:\n\n- Make sure dummy interface exists in the node\n- Bind service cluster IP to the dummy interface\n- Create an ipvs service whose address corresponding to kubernetes service Cluster IP\n- Iterate LB's ingress IPs, create an ipvs service whose address corresponding LB's ingress IP\n\nFor example,\n\n```shell\n# kubectl describe svc nginx-service\nName:\t\t\tnginx-service\n...\nIP:\t\t\t    10.102.128.4\nPort:\t\t\thttp\t3080/TCP\nEndpoints:\t\t10.244.0.235:8080\nSession Affinity:\tNone\n\n#### Only bind Cluster IP to dummy interface\n# ip addr\n...\n73: kube-ipvs0: \u003cBROADCAST,NOARP\u003e mtu 1500 qdisc noop state DOWN qlen 1000\n    link/ether 1a:ce:f5:5f:c1:4d brd ff:ff:ff:ff:ff:ff\n    inet 10.102.128.4/32 scope global kube-ipvs0\n       valid_lft forever preferred_lft forever\n\n#### Suppose LB's ingress IPs {10.96.1.2, 10.93.1.3}. IPVS proxier will create 1 ipvs service for cluster IP and 2 ipvs services for LB's ingree IP. Each ipvs service has its destination.\n# ipvsadm -ln\nIP Virtual Server version 1.2.1 (size=4096)\nProt LocalAddress:Port Scheduler Flags\n  -\u003e RemoteAddress:Port           Forward Weight ActiveConn InActConn     \nTCP  10.102.128.4:3080 rr\n  -\u003e 10.244.0.235:8080            Masq    1      0          0           \nTCP  10.96.1.2:3080 rr  \n  -\u003e 10.244.0.235:8080            Masq    1      0          0   \nTCP  10.96.1.3:3080 rr  \n  -\u003e 10.244.0.235:8080            Masq    1      0          0   \n```\n\nSince there is a need of supporting access control for `LB.ingress.IP`. IPVS proxier will fall back on iptables. Iptables will drop any packet which is not from `LB.LoadBalancerSourceRanges`. For example,\n\n```shell\n# iptables -A KUBE-SERVICES -d {ingress.IP} --dport {service.Port} -s {LB.LoadBalancerSourceRanges} -j ACCEPT\n```\n\nWhen the packet reach the end of chain, ipvs proxier will drop it.\n\n```shell\n# iptables -A KUBE-SERVICES -d {ingress.IP} --dport {service.Port} -j KUBE-MARK-DROP\n```\n\n### Support Only NodeLocal Endpoints\n\nSimilar to iptables proxier, when a service has the \"Only NodeLocal Endpoints\" annotation,  ipvs proxier will only proxy traffic to endpoints in the local node.\n\n```shell\n# kubectl describe svc nginx-service\nName:\t\t\tnginx-service\n...\nIP:\t\t\t    10.102.128.4\nPort:\t\t\thttp\t3080/TCP\nEndpoints:\t\t10.244.0.235:8080, 10.244.1.235:8080\nSession Affinity:\tNone\n\n#### Assume only endpoint 10.244.0.235:8080 is in the same host with kube-proxy\n\n#### There should be 1 destination for ipvs service.\n[root@SHA1000130405 home]# ipvsadm -ln\nIP Virtual Server version 1.2.1 (size=4096)\nProt LocalAddress:Port Scheduler Flags\n  -\u003e RemoteAddress:Port           Forward Weight ActiveConn InActConn     \nTCP  10.102.128.4:3080 rr\n  -\u003e 10.244.0.235:8080            Masq    1      0          0               \n```\n\n#### Session affinity\n\nIPVS support client IP session affinity (persistent connection). When a service specifies session affinity, the IPVS proxier will set a timeout value (180min=10800s by default) in the IPVS service. For example,\n\n```shell\n# kubectl describe svc nginx-service\nName:\t\t\tnginx-service\n...\nIP:\t\t\t    10.102.128.4\nPort:\t\t\thttp\t3080/TCP\nSession Affinity:\tClientIP\n\n# ipvsadm -ln\nIP Virtual Server version 1.2.1 (size=4096)\nProt LocalAddress:Port Scheduler Flags\n  -\u003e RemoteAddress:Port           Forward Weight ActiveConn InActConn\nTCP  10.102.128.4:3080 rr persistent 10800\n```\n\n#### Cleaning up inactive rules\n\nIt seems difficult to distinguish if an IPVS service is created by the IPVS proxier or other processes. Currently we assume IPVS rules will be created only by the IPVS proxier on a node, so we can clear all IPVSrules on a node. We should add warnings in documentation and flag comments.\n\n#### Sync loop pseudo code\n\nSimilar to the iptables proxier, the IPVS proxier will do a full sync loop in a configured period. Also, each update on a Kubernetes service or endpoint will trigger an IPVS service or destination update. For example,\n\n* Creating a Kubernetes service will trigger creating a new IPVS service.\n* Updating a Kubernetes service(for instance, change session affinity) will trigger updating an existing IPVS service.\n* Deleting a Kubernetes service will trigger deleting an IPVS service.\n* Adding an endpoint for a Kubernetes service will trigger adding a destination for an existing IPVS service.\n* Updating an endpoint for a Kubernetes service will trigger updating a destination for an existing IPVS service.\n* Deleting an endpoint for a Kubernetes service will trigger deleting a destination for an existing IPVS service.\n\nAny IPVS service or destination updates will send an update command to kernel via socket communication, which won't take a service down.\n\nThe sync loop pseudo code is shown below:\n\n```go\nfunc (proxier *Proxier) syncProxyRules() {\n\tWhen service or endpoint update, begin sync ipvs rules and iptables rules if needed.\n    ensure dummy interface exists, if not, create one.\n    for svcName, svcInfo := range proxier.serviceMap {\n      // Capture the clusterIP.\n      construct ipvs service from svcInfo\n      Set session affinity flag and timeout value for ipvs service if specified session affinity\n      bind Cluster IP to dummy interface\n      call libnetwork API to create ipvs service and destinations\n\n      // Capture externalIPs.\n      if externalIP is local then hold the svcInfo.Port so that can install ipvs rules on it\n      construct ipvs service from svcInfo\n      Set session affinity flag and timeout value for ipvs service if specified session affinity\n      call libnetwork API to create ipvs service and destinations\n\n      // Capture load-balancer ingress.\n\t    for _, ingress := range svcInfo.LoadBalancerStatus.Ingress {\n\t\t    if ingress.IP != \"\" {\n          if len(svcInfo.LoadBalancerSourceRanges) != 0 {\n            install specific iptables\n          }\n          construct ipvs service from svcInfo\n          Set session affinity flag and timeout value for ipvs service if specified session affinity\n          call libnetwork API to create ipvs service and destinations\n        }\n      }\n\n      // Capture nodeports.\n      if svcInfo.NodePort != 0 {\n\t\t    fall back on iptables, recruit existing iptables proxier implementation\n      }\n\n      call libnetwork API to clean up legacy ipvs services which is inactive any longer\n      unbind service address from dummy interface\n      clean up legacy iptables chains and rules\n    }\n}\n```\n\n## Graduation Criteria\n\n### Beta -\u003e GA\n\nThe following requirements should be met before moving from Beta to GA. It is\nsuggested to file an issue which tracks all the action items.\n\n- [ ] Testing\n    - [ ] 48 hours of green e2e tests. \n    - [ ] Flakes must be identified and filed as issues.\n    - [ ] Integrate with scale tests and. Failures should be filed as issues.\n- [ ] Development work\n    - [ ] Identify all pending changes/refactors. Release blockers must be prioritized and fixed.\n    - [ ] Identify all bugs. Release blocking bugs must be identified and fixed.\n- [ ] Docs \n    - [ ] All user-facing documentation must be updated.\n\n### GA -\u003e Future\n\n__TODO__\n\n## Implementation History\n\n**In chronological order**\n\n1. https://github.com/kubernetes/kubernetes/pull/46580\n\n2. https://github.com/kubernetes/kubernetes/pull/52528\n\n3. https://github.com/kubernetes/kubernetes/pull/54219\n\n4. https://github.com/kubernetes/kubernetes/pull/57268\n\n5. https://github.com/kubernetes/kubernetes/pull/58052\n\n\n## Drawbacks [optional]\n\nNone\n\n## Alternatives [optional]\n\nNone\n"
+  },
+  {
+    "id": "113f600150fbd3502face2b1b0d8162d",
+    "title": "Switch CoreDNS to the default DNS",
+    "authors": ["@johnbelamaric", "@rajansandeep"],
+    "owningSig": "sig-network",
+    "participatingSigs": ["sig-cluster-lifecycle"],
+    "reviewers": ["@bowei", "@thockin"],
+    "approvers": ["@thockin"],
+    "editor": "@rajansandeep",
+    "creationDate": "2018-05-18",
+    "lastUpdated": "2018-05-18",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Switch CoreDNS to the default DNS\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Goals](#goals)\n- [Proposal](#proposal)\n  - [Use Cases](#use-cases)\n    - [Kubeadm](#kubeadm)\n    - [Kube-up](#kube-up)\n    - [Minikube](#minikube)\n    - [Kops](#kops)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nCoreDNS is now well-established in Kubernetes as the DNS service, with CoreDNS starting as an alpha feature from Kubernetes v1.9 to now being GA in v1.11.\nAfter successfully implementing the road-map defined [here](https://github.com/kubernetes/features/issues/427), CoreDNS is GA in Kubernetes v1.11, which can be installed as an alternate to kube-dns in tools like kubeadm, kops, minikube and kube-up.\nFollowing the [KEP to graduate CoreDNS to GA](https://github.com/kubernetes/community/pull/1956), the purpose of this proposal is to make CoreDNS as the default DNS for Kubernetes, replacing kube-dns.\n\n## Goals\n* Make CoreDNS the default DNS for Kubernetes for all the remaining install tools (kube-up, kops, minikube).\n* Make CoreDNS available as an image in a Kubernetes repository (To Be Defined) and ensure a workflow/process to update the CoreDNS versions in the future.\n  This goal is carried over from the [previous KEP](https://github.com/kubernetes/community/pull/1956), in case it cannot be completed there.\n\n## Proposal\n\nThe proposed solution is to enable CoreDNS as the default cluster service discovery DNS for Kubernetes.\nSome of the most used deployment tools will be upgraded by the CoreDNS team, in cooperation with the owners of these tools, to be able to deploy CoreDNS as default:\n* kubeadm (already done for Kubernetes v1.11)\n* kube-up\n* minikube\n* kops\n\nFor other tools, each maintainer would have to add the upgrade to CoreDNS.\n\n### Use Cases\n\nUse cases for CoreDNS has been well defined in the [previous KEP](https://github.com/kubernetes/community/pull/1956).\nThe following can be expected when CoreDNS is made the default DNS.\n\n#### Kubeadm\n\n* CoreDNS is already the default DNS from Kubernetes v1.11 and shall continue be the default DNS.\n* In case users want to install kube-dns instead of CoreDNS, they have to set the feature-gate of CoreDNS to false. `--feature-gates=CoreDNS=false`\n\n#### Kube-up\n\n* CoreDNS will now become the default DNS.\n* To install kube-dns in place of CoreDNS, set the environment variable `CLUSTER_DNS_CORE_DNS` to `false`.\n\n#### Minikube\n\n* CoreDNS to be enabled by default in the add-on manager, with kube-dns disabled by default.\n\n#### Kops\n\n* CoreDNS will now become the default DNS.\n\n## Graduation Criteria\n\n* Add CoreDNS image in a Kubernetes repository (To Be Defined) and ensure a workflow/process to update the CoreDNS versions in the future.\n* Have a certain number (To Be Defined) of clusters of significant size (To Be Defined) adopting and running CoreDNS as their default DNS.\n\n## Implementation History\n\n* 20170912 - [Feature proposal](https://github.com/kubernetes/features/issues/427) for CoreDNS to be implemented as the default DNS in Kubernetes.\n* 20171108 - Successfully released [CoreDNS as an Alpha feature-gate in Kubernetes v1.9](https://github.com/kubernetes/kubernetes/pull/52501).\n* 20180226 - CoreDNS graduation to Incubation in CNCF.\n* 20180305 - Support for Kube-dns configmap translation and move up [CoreDNS to Beta](https://github.com/kubernetes/kubernetes/pull/58828) for Kubernetes v1.10.\n* 20180515 - CoreDNS was added as [GA and the default DNS in kubeadm](https://github.com/kubernetes/kubernetes/pull/63509) for Kubernetes v1.11.\n"
+  },
+  {
+    "id": "707d15e1a9558a984c66f253cec3f80a",
+    "title": "SCTP support",
+    "authors": ["@janosi"],
+    "owningSig": "sig-network",
+    "participatingSigs": ["sig-network"],
+    "reviewers": ["@thockin"],
+    "approvers": ["@thockin"],
+    "editor": "@danwinship",
+    "creationDate": "2018-06-14",
+    "lastUpdated": "2019-09-20",
+    "status": "implementable",
+    "seeAlso": ["PR64973"],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# SCTP support\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories [optional]](#user-stories-optional)\n    - [Service with SCTP and Virtual IP](#service-with-sctp-and-virtual-ip)\n    - [Headless Service with SCTP](#headless-service-with-sctp)\n    - [Service with SCTP without selector](#service-with-sctp-without-selector)\n    - [SCTP as container port protocol in Pod definition](#sctp-as-container-port-protocol-in-pod-definition)\n    - [SCTP port accessible from outside the cluster](#sctp-port-accessible-from-outside-the-cluster)\n    - [NetworkPolicy with SCTP](#networkpolicy-with-sctp)\n    - [Userspace SCTP stack](#userspace-sctp-stack)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n    - [SCTP in Services](#sctp-in-services)\n      - [Kubernetes API modification](#kubernetes-api-modification)\n      - [Services with host level ports](#services-with-host-level-ports)\n      - [Services with type=LoadBalancer](#services-with-typeloadbalancer)\n    - [SCTP support in Kube DNS](#sctp-support-in-kube-dns)\n    - [SCTP in the Pod's ContainerPort](#sctp-in-the-pods-containerport)\n    - [SCTP in NetworkPolicy](#sctp-in-networkpolicy)\n    - [Interworking with applications that use a user space SCTP stack](#interworking-with-applications-that-use-a-user-space-sctp-stack)\n      - [Problem definition](#problem-definition)\n      - [The solution in the Kubernetes SCTP support implementation](#the-solution-in-the-kubernetes-sctp-support-implementation)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Test Plan](#test-plan)\n  - [Basic tests](#basic-tests)\n  - [SCTP Connectivity Tests](#sctp-connectivity-tests)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThe goal of the SCTP support feature is to enable the usage of the SCTP protocol in Kubernetes [Service][],  [NetworkPolicy][], and [ContainerPort][]as an additional protocol value option beside the current TCP and UDP options.\nSCTP is an IETF protocol specified in [RFC4960][], and it is used widely in telecommunications network stacks.\nOnce SCTP support is added as a new protocol option those applications that require SCTP as L4 protocol on their interfaces can be deployed on Kubernetes clusters on a more straightforward way. For example they can use the native kube-dns based service discovery, and their communication can be controlled on the native NetworkPolicy way.\n\n[Service]: https://kubernetes.io/docs/concepts/services-networking/service/\n[NetworkPolicy]: \nhttps://kubernetes.io/docs/concepts/services-networking/network-policies/\n[ContainerPort]:https://kubernetes.io/docs/concepts/services-networking/connect-applications-service/#exposing-pods-to-the-cluster\n[RFC4960]: https://tools.ietf.org/html/rfc4960\n\n\n## Motivation\n\nSCTP is a widely used protocol in telecommunications. It would ease the management and execution of telecommunication applications on Kubernetes if SCTP were added as a protocol option to Kubernetes. \n\n### Goals\n\nAdd SCTP support to Kubernetes ContainerPort, Service and NetworkPolicy, so applications running in pods can use the native kube-dns based service discovery for SCTP based services, and their communication can be controlled via the native NetworkPolicy way.\n\nIt is also a goal to enable ingress SCTP connections from clients outside the Kubernetes cluster, and to enable egress SCTP connections to servers outside  the Kubernetes cluster.\n\n### Non-Goals\n\nIt is not a goal here to add SCTP support to load balancers that are provided by cloud providers. The Kubernetes side implementation will not restrict the usage of SCTP as the protocol for the Services with type=LoadBalancer, but we do not implement the support of SCTP into the cloud specific load balancer implementations.\n\nIt is not a goal to support multi-homed SCTP associations. Such a support also depends on the ability to manage multiple IP addresses for a pod, and in the case of Services with ClusterIP or NodePort the support of multi-homed associations would also require the support of NAT for multihomed associations in the SCTP related NF conntrack modules.\n\n## Proposal\n\n### User Stories [optional]\n\n#### Service with SCTP and Virtual IP\nAs a user of Kubernetes I want to define Services with Virtual IPs for my applications that use SCTP as L4 protocol on their interfaces,so client applications can use the services of my applications on top of SCTP via that Virtual IP. \n\nExample:\n```\nkind: Service\napiVersion: v1\nmetadata:\n  name: my-service\nspec:\n  selector:\n    app: MyApp\n  ports:\n  - protocol: SCTP\n    port: 80\n    targetPort: 9376\n```\n\n#### Headless Service with SCTP\nAs a user of Kubernetes I want to define headless Services for my applications that use SCTP as L4 protocol on their interfaces, so client applications can discover my applications in kube-dns, or via any other service discovery methods that get information about endpoints via the Kubernetes API.\n\nExample:\n```\nkind: Service\napiVersion: v1\nmetadata:\n  name: my-service\nspec:\n  selector:\n    app: MyApp\n  ClusterIP: \"None\"\n  ports:\n  - protocol: SCTP\n    port: 80\n    targetPort: 9376\n```\n#### Service with SCTP without selector\nAs a user of Kubernetes I want to define Services without selector for my applications that use SCTP as L4 protocol on their interfaces, so I can implement my own service controllers if I want to extend the basic functionality of Kubernetes.\n\nExample:\n```\nkind: Service\napiVersion: v1\nmetadata:\n  name: my-service\nspec:\n  ClusterIP: \"None\"\n  ports:\n  - protocol: SCTP\n    port: 80\n    targetPort: 9376\n```\n\n#### SCTP as container port protocol in Pod definition\nAs a user of Kubernetes I want to define hostPort for the SCTP based interfaces of my applications\nExample:\n```\napiVersion: v1\nkind: Pod\nmetadata:\n  name: mypod\nspec:\n  containers:\n  - name: container-1\n    image: mycontainerimg\n    ports:\n      - name: diameter\n        protocol: SCTP\n        containerPort: 80\n        hostPort: 80\n```\n\n#### SCTP port accessible from outside the cluster\n\nAs a user of Kubernetes I want to have the option that client applications that reside outside of the cluster can access my SCTP based services that run in the cluster.\n\nExample:\n```\nkind: Service\napiVersion: v1\nmetadata:\n  name: my-service\nspec:\n  type: NodePort\n  selector:\n    app: MyApp\n  ports:\n  - protocol: SCTP\n    port: 80\n    targetPort: 9376\n```\n\nExample:\n```\nkind: Service\napiVersion: v1\nmetadata:\n  name: my-service\nspec:\n  selector:\n    app: MyApp\n  ports:\n  - protocol: SCTP\n    port: 80\n    targetPort: 9376\n  externalIPs:\n  - 80.11.12.10\n```\n\n#### NetworkPolicy with SCTP\nAs a user of Kubernetes I want to define NetworkPolicies for my applications that use SCTP as L4 protocol on their interfaces, so the network plugins that support SCTP can control the accessibility of my applications on the SCTP based interfaces, too.\n\nExample:\n```\napiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\n  name: myservice-network-policy\n  namespace: myapp\nspec:\n  podSelector:\n    matchLabels:\n      role: myservice\n  policyTypes:\n  - Ingress\n  ingress:\n  - from:\n    - ipBlock:\n        cidr: 172.17.0.0/16\n        except:\n        - 172.17.1.0/24\n    - namespaceSelector:\n        matchLabels:\n          project: myproject\n    - podSelector:\n        matchLabels:\n          role: myclient\n    ports:\n    - protocol: SCTP\n      port: 7777\n```\n#### Userspace SCTP stack\nAs a user of Kubernetes I want to deploy and run my applications that use a userspace SCTP stack, and at the same time I want to define SCTP Services in the same cluster. I use a userspace SCTP stack because of the limitations of the kernel's SCTP support. For example: it's not possible to write an SCTP server that proxies/filters arbitrary SCTP streams using the sockets APIs and kernel SCTP.\n\n### Implementation Details/Notes/Constraints [optional]\n\n#### SCTP in Services\n##### Kubernetes API modification\nThe Kubernetes API modification for Services to support SCTP is obvious.\n\n##### Services with host level ports\n\nThe kube-proxy and the kubelet starts listening on the defined TCP or UDP port in case of Servies with ClusterIP or NodePort or externalIP, and in case of containers with HostPort defined. The goal of this is to reserve the port in question so no other host level process can use that by accident. When it comes to SCTP the agreement is that we do not follow this pattern. That is, Kubernetes will not listen on host level ports with SCTP as protocol. The reason for this decision is, that  the current TCP and UDP related implementation is not perfect either, it has known gaps in some use cases, and in those cases this listening is not started. But no one complained about those gaps so most probably this port reservation via listening logic is not needed at all.\n\n##### Services with type=LoadBalancer\n\nFor Services with type=LoadBalancer we expect that the cloud provider's load balancer API client in Kubernetes rejects the requests with unsupported protocol.\n\n#### SCTP support in Kube DNS \nKube DNS shall support SRV records with \"_sctp\" as \"proto\" value. According to our investigations, the DNS controller is very flexible from this perspective, and it can create SRV records with any protocol name. I.e. there is no need for additional implementation to achieve this goal.\n\nExample:\n\n```\n_diameter._sctp.my-service.default.svc.cluster.local. 30 IN SRV 10 100 1234 my-service.default.svc.cluster.local.\n```\n#### SCTP in the Pod's ContainerPort\nThe Kubernetes API modification for the Pod is obvious.\n\nWe support SCTP as protocol for any combinations of containerPort and hostPort.\n\n#### SCTP in NetworkPolicy\nThe Kubernetes API modification for the NetworkPolicy is obvious.\n\nIn order to utilize the new protocol value the network plugin must support it.\n\n#### Interworking with applications that use a user space SCTP stack\n\n##### Problem definition\nA userpace SCTP stack usually creates raw sockets with IPPROTO_SCTP. And as it is clearly highlighted in the [documentation of raw sockets][]:\n\u003eRaw sockets may tap all IP protocols in Linux, even protocols like ICMP or TCP which have a protocol module in the kernel.  In this case, the packets are passed to both the kernel module and the raw socket(s).\n\nI.e. if both the kernel module (lksctp) and a userspace SCTP stack are active on the same node both receive the incoming SCTP packets according to the current [kernel][] logic.\n\nBut in turn the SCTP kernel module will handle those packets that are actually destined to the raw socket as Out of the blue (OOTB) packets according to the rules defined in [RFC4960][]. I.e. the SCTP kernel module sends SCTP ABORT to the sender, and on that way it aborts the connections of the userspace SCTP stack.\n\nAs we can see, a userspace SCTP stack cannot co-exist with the SCTP kernel module (lksctp) on the same node. That is, the loading of the SCTP kernel module must be avoided on nodes where such applications that use userspace SCTP stack are planned to be run. The SCTP kernel module loading is triggered when an application starts managing SCTP sockets via the standard socket API or via syscalls.\n\nIn order to resolve this problem the solution was to dedicate nodes to userspace SCTP applications in the past. Such applications that would trigger the loading of the SCTP kernel module were not deployed on those nodes.\n\n##### The solution in the Kubernetes SCTP support implementation\nOur main task here is to provide the same node level isolation possibility that was used in the past: i.e. to provide the option to dedicate some nodes to userspace SCTP applications, and ensure that the actions performed by Kubernetes (kubelet, kube-proxy) do not load the SCTP kernel modules on those dedicated nodes.\n\nOn the Kubernetes side we solve this problem so, that we do not start listening on the SCTP ports defined for Servies with ClusterIP or NodePort or externalIP, neither in the case when containers with SCTP HostPort are defined. On this way we avoid the loading of the kernel module due to Kubernetes actions.\n\nOn application side it is pretty easy to separate application pods that use a userspace SCTP stack from those application pods that use the kernel space SCTP stack: the usual nodeselector label based mechanism, or taints are there for this very purpose. \n\nNOTE! The handling of TCP and UDP Services does not change on those dedicated nodes.\n\nWe propose the following solution:\n\nWe describe in the Kubernetes documentation the mutually exclusive nature of userspace and kernel space SCTP stacks, and we would highlight, that the required separation of the userspace SCTP stack applications and the kernel module users shall be achieved with the usual nodeselector or taint based mechanisms.\n\n\n[documentation of raw sockets]: http://man7.org/linux/man-pages/man7/raw.7.html\n[kernel]: https://github.com/torvalds/linux/blob/0fbc4aeabc91f2e39e0dffebe8f81a0eb3648d97/net/ipv4/ip_input.c#L191\n\n### Risks and Mitigations\n\n## Test Plan\n\nUnlike with UDP and TCP, we can't necessarily just test that SCTP\nconnections work, because some machines won't have the SCTP kernel\nmodule installed (or will have it blocked from being loaded). So we\nwant to have some tests that are just \"make sure kube-proxy, etc are\ndoing what we expect them to\" that can run everywhere, and then\nanother set of \"SCTP connections actually work\" tests that are behind\na separate feature tag.\n\n### Basic tests\n\nThese will be tagged `[Feature:SCTP]`.\n\n- `\"Pods, Services, and NetworkPolicies can declare SCTP ports\"`\n\n  - Just checks that resources can be created using `SCTP` as a\n    `Protocol` value.\n\n- `\"Pods can declare SCTP HostPorts when using kubenet\"`\n\n  - Confirms that adding an SCTP HostPort creates an appropriate\n    iptables rule.\n\n  - Will be Skipped if SSH is not available or kubenet is not in use.\n\n- `\"Services can declare SCTP ServicePorts when using the iptables proxier\"`\n\n  - Confirms that appropriate iptables rules are created for SCTP\n    services.\n\n  - Will be Skipped if SSH is not available or the `iptables` proxier\n    is not in use.\n\n- `\"A NetworkPolicy matching an SCTP port does not allow TCP or UDP\n  traffic [Feature:NetworkPolicy]\"`\n\nIf SSH is available, then each test will also ensure that if `sctp.ko`\nis not loaded before the test, then it is also not loaded after the\ntest.\n\nSince the tests will initially require the `SCTPSupport` feature gate\nto be enabled, we will create a periodic job to run just the SCTP\ntests (and perhaps a few other network conformance tests, just to\nconfirm that enabling SCTP does not break other things.) After SCTP\nreaches GA, the tests could run as part of the normal presubmit job\nand the periodic job could be retired.\n\n### SCTP Connectivity Tests\n\nThese will be tagged \"`[Feature:SCTPConnectivity]`\". They can be run\nby network plugin developers but will not run as part of any of the\nstandard Kubernetes test runs. They should all be marked\n`[Disruptive]` since they may cause the SCTP kernel module to be\nloaded, which may interfere with the Basic Tests' \"make sure the SCTP\nkernel module doesn't get loaded\" checks.\n\nWe will need to vendor\n[github.com/ishidawataru/sctp](https://github.com/ishidawataru/sctp)\nand use it from `agnhost` and `e2e.test` to create SCTP client and\nserver sockets.\n\n- `\"A pod can connect directly to another pod via SCTP\"`\n\n- `\"A pod can connect to another pod via SCTP via a Service IP\"`\n\n- `\"A pod can connect to another pod via SCTP via a Load Balancer\"`,\n\n  - Will be Skipped on bare metal, and clouds that don't support SCTP\n    LoadBalancers.\n\n- `\"NetworkPolicy can be used to allow or deny traffic to SCTP ports\n  [Feature:NetworkPolicy]\"`\n\n## Graduation Criteria\n\n### Alpha -\u003e Beta Graduation\n\nGraduation criteria:\n\n- The e2e tests described in the Test Plan have been written.\n\n- A periodic job has been created to run the \"Basic Tests\", and it\n  passes.\n\n### Beta -\u003e GA Graduation\n\nGraduation criteria:\n\n- At least 2 out-of-tree network plugins can pass the\n  `[Feature:SCTPConnectivity]` tests.\n\n- Both `iptables` and `ipvs` proxiers have been demonstrated to work\n  with SCTP.\n\n## Implementation History\n\n- 2018-06-11 Initial [code PR](https://github.com/kubernetes/kubernetes/pull/64973)\n- 2018-06-16 Initial [KEP PR](https://github.com/kubernetes/community/pull/2276)\n- 2018-08-24 Initial KEP merged\n- 2018-08-28 Initial code merged\n- 2018-09-11 [Feature proposal](https://github.com/kubernetes/enhancements/issues/614) filed\n- 2018-09-27 Kubernetes 1.12.0 released with Alpha SCTP support\n- 2019-10-02 Test Plan and Graduation Criteria added\n"
+  },
+  {
+    "id": "f4a600323b408d249edfd6623cf2d9d6",
+    "title": "NodeLocal DNS Cache",
+    "authors": ["@prameshj"],
+    "owningSig": "sig-network",
+    "participatingSigs": ["sig-network"],
+    "reviewers": ["@thockin", "@bowei", "@johnbelamaric", "@sdodson"],
+    "approvers": ["@thockin", "@bowei"],
+    "editor": "TBD",
+    "creationDate": "2018-10-05",
+    "lastUpdated": "2019-04-25",
+    "status": "implemented",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# NodeLocal DNS Cache\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n    - [Daemonset and Listen Interface for caching agent](#daemonset-and-listen-interface-for-caching-agent)\n    - [iptables NOTRACK](#iptables-notrack)\n    - [Choice of caching agent](#choice-of-caching-agent)\n    - [Metrics](#metrics)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Rollout Plan](#rollout-plan)\n- [Implementation History](#implementation-history)\n- [Drawbacks [optional]](#drawbacks-optional)\n- [Alternatives [optional]](#alternatives-optional)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis proposal aims to improve DNS performance by running a dns caching agent on cluster nodes as a Daemonset. In today's architecture, pods in ClusterFirst DNS mode reach out to a kube-dns serviceIP for DNS queries. This is translated to a kube-dns endpoint via iptables rules added by kube-proxy. With this new architecture, pods will reach out to the dns caching agent running on the same node, thereby avoiding iptables DNAT rules and connection tracking. The local caching agent will query kube-dns for cache misses of cluster hostnames(cluster.local suffix by default).\n\n\n## Motivation\n\n* With the current DNS architecture, it is possible that pods with the highest DNS QPS have to reach out to a different node, if there is no local kube-dns instance.  \nHaving a local cache will help improve the latency in such scenarios. \n\n* Skipping iptables DNAT and connection tracking will help reduce [conntrack races](https://github.com/kubernetes/kubernetes/issues/56903) and avoid UDP DNS entries filling up conntrack table.\n\n* Connections from local caching agent to kube-dns can be upgraded to TCP. TCP conntrack entries will be removed on connection close in contrast with UDP entries that have to timeout ([default](https://www.kernel.org/doc/Documentation/networking/nf_conntrack-sysctl.txt) `nf_conntrack_udp_timeout` is 30 seconds)\n\n* Upgrading DNS queries from UDP to TCP would reduce tail latency attributed to dropped UDP packets and DNS timeouts usually up to 30s (3 retries + 10s timeout). Since the nodelocal cache listens for UDP DNS queries, applications don't need to be changed.\n\n* Metrics \u0026 visibility into dns requests at a node level.\n\n* Neg caching can be re-enabled, thereby reducing number of queries to kube-dns.\n\n* There are several open github issues proposing a local DNS Cache daemonset and scripts to run it:\n\t* [https://github.com/kubernetes/kubernetes/issues/7470#issuecomment-248912603](https://github.com/kubernetes/kubernetes/issues/7470#issuecomment-248912603)\n\n\t* [https://github.com/kubernetes/kubernetes/issues/32749](https://github.com/kubernetes/kubernetes/issues/32749)\n\n\t* [https://github.com/kubernetes/kubernetes/issues/45363](https://github.com/kubernetes/kubernetes/issues/45363)\n\n\nThis shows that there is interest in the wider Kubernetes community for a solution similar to the proposal here. \n\n\n### Goals\n\nBeing able to run a dns caching agent as a Daemonset and get pods to use the local instance. Having visibility into cache stats and other metrics.\n\n### Non-Goals\n\n* Providing a replacement for kube-dns/CoreDNS.\n* Changing the underlying protocol for DNS (e.g. to gRPC)\n\n## Proposal\n\nA nodeLocal dns cache runs on all cluster nodes. This is managed as an add-on, runs as a Daemonset. All pods using clusterDNS will now talk to the nodeLocal cache, which will query kube-dns in case of cache misses in cluster's configured DNS suffix and for all reverse lookups(in-addr.arpa and ip6.arpa). User-configured stubDomains will be passed on to this local agent.  \nThe node's resolv.conf will be used by this local agent for all other cache misses. One benefit of doing the non-cluster lookups on the nodes from which they are happening, rather than the kube-dns instances, is better use of per-node DNS resources in cloud. For instance, in a 10-node cluster with 3 kube-dns instances, the 3 nodes running kube-dns will end up resolving all external hostnames and can exhaust QPS quota. Spreading the queries across the 10 nodes will help alleviate this.\n\n#### Daemonset and Listen Interface for caching agent\n\nThe caching agent daemonset runs in hostNetwork mode in kube-system namespace with a Priority Class of “system-node-critical”. It listens for dns requests on a dummy interface created on the host. A separate ip address is assigned to this dummy interface, so that requests to kube-dns or any other custom service are not incorrectly intercepted by the caching agent. This will be a link-local ip address selected by the user. Each cluster node will have this dummy interface. This ip address will be passed on to kubelet via the --cluster-dns flag, if the feature is enabled.\n\nThe selected link-local IP will be handled specially because of the NOTRACK rules described in the section below.\n\n#### iptables NOTRACK\n\nNOTRACK rules are added for connections to and from the nodelocal dns ip. Additional rules in FILTER table to whitelist these connections, since the INPUT and OUTPUT chains have a default DROP policy.\n\nThe nodelocal cache process will create the dummy interface and iptables rules . It gets the nodelocal dns ip as a parameter, performs setup and listens for dns requests. The Daemonset runs in privileged securityContext since it needs to create this dummy interface and add iptables rules.\n The cache process will also periodically ensure that the dummy interface and iptables rules are present, in the background. Rules need to be checked in the raw table and filter table. Rules in these tables do not grow with number of valid services. Services with no endpoints will have rules added in filter table to drop packets destined to these ip. The resource usage for periodic iptables check was measured by creating 2k services with no endpoints and running the nodelocal caching agent. Peak memory usage was 20Mi for the caching agent when it was responding to queries along with the periodic checks. This was measured using `kubectl top` command. More details on the testing are in the following section.\n\n[Proposal presentation](https://docs.google.com/presentation/d/1c43cZqbVhGAlw3dSNQIOGuvQmDfKaA2yiAPRoYpa6iY), also shared at the sig-networking meeting on 2018-10-04\n\nSlide 5 has a diagram showing how the new dns cache fits in.\n\n#### Choice of caching agent\n\nThe current plan is to run CoreDNS by default. Benchmark [ tests](https://github.com/kubernetes/perf-tests/tree/master/dns) were run using [Unbound dns server](https://www.nlnetlabs.nl/projects/unbound/about/) and CoreDNS. 2 more tests were added to query for 20 different services and to query several external hostnames.\n\nTests were run on a 1.9.7 cluster with 2 nodes on GCE, using Unbound 1.7.3 and CoreDNS 1.2.3.\nResource limits for nodelocaldns daemonset was CPU - 50m, Memory 25Mi\n\nResource usage and QPS were measured with a nanny process for Unbound/CoreDNS plugin adding iptables rules and ensuring that the rules exist, every minute.\n\nCaching was minimized in Unbound by setting:\nmsg-cache-size: 0\nrrset-cache-size: 0\nmsg-cache-slabs:1\nrrset-cache-slabs:1\nPrevious tests did not set the last 2 and there were quite a few unexpected cache hits.\n\nCaching was disabled in CoreDNS by skipping the cache plugin from Corefile.\n\nThese are the results when dnsperf test was run with no QPS limit. In this mode, the tool  sends queries until they start timing out.\n\n| Test Type             | Program | Caching | QPS  |\n| Multiple services(20) | CoreDNS | Yes     | 860  |\n| Multiple services(20) | Unbound | Yes     | 3030 |\n|                       |         |         |      |\n| External queries      | CoreDNS | Yes     | 213  |\n| External queries      | Unbound | Yes     | 115  |\n|                       |         |         |      |\n| Single Service        | CoreDNS | Yes     | 834  |\n| Single Service        | Unbound | Yes     | 3287 |\n|                       |         |         |      |\n| Single NXDomain       | CoreDNS | Yes     | 816  |\n| Single NXDomain       | Unbound | Yes     | 3136 |\n|                       |         |         |      |\n| Multiple services(20) | CoreDNS | No      | 859  |\n| Multiple services(20) | Unbound | No      | 1463 |\n|                       |         |         |      |\n| External queries      | CoreDNS | No      | 180  |\n| External queries      | Unbound | No      | 108  |\n|                       |         |         |      |\n| Single Service        | CoreDNS | No      | 818  |\n| Single Service        | Unbound | No      | 2992 |\n|                       |         |         |      |\n| Single NXDomain       | CoreDNS | No      | 827  |\n| Single NXDomain       | Unbound | No      | 2986 |\n\n\nPeak memory usage was ~20 Mi for both Unbound and CoreDNS.\n\nFor the single service and single NXDomain query, Unbound still had cache hits since caching could not be completely disabled.\n\nCoreDNS QPS was twice as much as Unbound for external queries. They were mostly unique hostnames from this file - [ftp://ftp.nominum.com/pub/nominum/dnsperf/data/queryfile-example-current.gz](ftp://ftp.nominum.com/pub/nominum/dnsperf/data/queryfile-example-current.gz)\n\nWhen multiple cluster services were queried with cache misses, Unbound was better(1463 vs 859), but not by a large factor.\n\nUnbound performs much better when all requests are cache hits.\n\nCoreDNS will be the local cache agent in the first release, after considering these reasons:\n\n*  Better QPS numbers for external hostname queries\n*  Single process, no need for a separate nanny process\n*  Prometheus metrics already available, also we can get per zone stats. Unbound gives consolidated stats.\n*  Easier to make changes to the source code\n\n It is possible to run any program as caching agent by modifying the daemonset and configmap spec. Publishing an image with Unbound DNS can be added as a follow up.\n\nBased on the prototype/test results, these are the recommended defaults: \nCPU request: 50m\nMemory Limit : 25m  \n\nCPU request can be dropped to a smaller value if QPS needs are lower.\n\n#### Metrics\n\nPer-zone metrics will be available via the metrics/prometheus plugin in CoreDNS.\n\n\n### Risks and Mitigations\n\nHaving the pods query the nodelocal cache introduces a single point of failure.\n\n* This is mitigated by having a livenessProbe to periodically ensure DNS is working. In case of upgrades, the recommendation is to drain the node before starting to upgrade the local instance. The user can also configure [customPodDNS](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-config) pointing to clusterDNS ip for pods that cannot handle DNS disruption during upgrade.\n\n* The Daemonset is assigned a PriorityClass of \"system-node-critical\", to ensure it is not evicted.\n\n* Populating both the nodelocal cache ip address and kube-dns ip address in resolv.conf is not a reliable option. Depending on underlying implementation, this can result in kube-dns being queried only if cache ip does not respond, or both queried simultaneously.\n\n\n## Graduation Criteria\nThis feature has been alpha since 1.13 release and beta since 1.15 release.\n\nGraduation criteria for GA(targeted for 1.18 release):\n- Upgrade to a newer CoreDNS version(1.6.x) in [node-cache](https://github.com/kubernetes/dns/pull/328).\n- Add a plan for periodic upgrade to newer CoreDNS versions.\n- Ensure that Kubernetes [e2e tests with NodeLocal DNSCache](https://k8s-testgrid.appspot.com/sig-network-gce#gci-gce-kube-dns-nodecache) are passing.\n- Scalability tests with NodeLocal DNSCache enabled, verifying the [HA modes](https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/20190424-NodeLocalDNS-beta-proposal.md#design-details) as well as the regular mode.\n- Add tests that clearly demonstrate the benefits of NodeLocal DNSCache and document the steps to run them.\n- Have 10 users running in production with NodeLocal DNSCache enabled.\n- Provide clear documentation on using NodeLocal DNSCache aimed at cluster\n  operators.\n\n## Rollout Plan\nThis feature will be launched with Alpha support in the first release. Master versions v1.13 and above will deploy the new add-on. Node versions v1.13 and above will have kubelet code to modify pods' resolv.conf. Nodes running older versions will run the nodelocal daemonset, but it will not be used. The user can specify a custom dnsConfig to use this local cache dns server.\n\n## Implementation History\n\n* 2018-10-05 - Creation of the KEP\n* 2018-10-30 - Follow up comments and choice of cache agent\n* 2018-11-14 - [Changes](https://github.com/kubernetes/kubernetes/pull/70555) to support running NodeLocal DNSCache were merged.\n* 2018-11-02 - Added GA graduation criteria\n\n## Drawbacks [optional]\n\nAdditional resource consumption for the Daemonset might not be necessary for clusters with low DNS QPS needs. \n\n\n## Alternatives [optional]\n\n* The listen ip address for the dns cache could be a service ip. This ip address is obtained by creating a nodelocaldns service, with same endpoints as the clusterDNS service. Using the same endpoints as clusterDNS helps reduce DNS downtime in case of upgrades/restart. When no other special handling is provided, queries to the nodelocaldns ip will be served by kube-dns/CoreDNS pods. Kubelet takes the service name as an argument `--cluster-dns-svc=\u003cnamespace\u003e/\u003csvc name\u003e`, looks up the ip address and populates pods' resolv.conf with this value instead of clusterDNS.\nThis approach works only for iptables mode of kube-proxy. This is because kube-proxy creates a dummy interface bound to all service IPs in ipvs mode and ipvs rules are added to load-balance between endpoints. The packet seems to get dropped if there are no endpoints. If there are endpoints, adding iptables rules does not bypass the ipvs loadbalancing rules.\n\n* A nodelocaldns service can be created with a hard requirement of same-node endpoint, once we have [this](https://github.com/kubernetes/community/pull/2846) supported. All the pods in the nodelocaldns daemonset will be endpoints, the one running locally will be selected. iptables rules to NOTRACK connections can still be added, in order to skip DNAT in the iptables kube-proxy implementation.\n\n* Instead of just a dns-cache, a full-fledged kube-dns instance can be run on all nodes. This will consume much more resources since each instance will also watch Services and Endpoints.\n"
+  },
+  {
+    "id": "71d3f9b69a3ac08e12d367be34f1663c",
+    "title": "Make kube-proxy service abstraction optional",
+    "authors": ["@bradhoekstra"],
+    "owningSig": "sig-network",
+    "participatingSigs": null,
+    "reviewers": ["@freehan"],
+    "approvers": ["@thockin"],
+    "editor": "@bradhoekstra",
+    "creationDate": "2018-10-17",
+    "lastUpdated": "2018-11-12",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Make kube-proxy service abstraction optional\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Story 1](#story-1)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n    - [Overview](#overview)\n    - [Design](#design)\n    - [Testing](#testing)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nIn a cluster that has a service mesh a lot of the work being done by kube-proxy is redundant and wasted.\nSpecifically, services that are only reached via other services in the mesh will never use the service abstraction implemented by kube-proxy in iptables (or ipvs).\nBy informing the kube-proxy of this, we can lighten the work it is doing and the burden on its proxy backend.\n\n## Motivation\n\nThe motivation for the enhancement is to allow higher scalability in large clusters with lots of services that are making use of a service mesh.\n\n### Goals\n\nThe goal is to reduce the load on:\n* The kube-proxy having to deserialize and process all services and endpoints\n* The backend system (e.g. iptables) for whichever proxy mode kube-proxy is using\n\n### Non-Goals\n\n* Making sure the service is still routable via the service mesh\n* Preserving any kube-proxy functionality for any intentionally disabled Service, including but not limited to: externalIPs, external LB routing, nodePorts, externalTrafficPolicy, healthCheckNodePort, UDP, SCTP\n\n## Proposal\n\n### User Stories\n\n#### Story 1\n\nAs a cluster operator, operating a cluster using a service mesh I want to be able to disable the kube-proxy service implementation for services in that mesh to reduce overall load on the whole cluster\n\n### Implementation Details/Notes/Constraints\n\n#### Overview\n\nIn a cluster where a service is only accessed via other applications in the service mesh the work that kube-proxy does to program the proxy (e.g. iptables) for that service is duplicated and unused. The service mesh itself handles load balancing for the service VIP. This case is often true in the standard service mesh setup of utilizing ingress/egress gateways, such that services are not directly exposed outside the cluster. In this setup, application services rarely make use of other Service features such as externalIPs, external LB routing, nodePorts, externalTrafficPolicy, healthCheckNodePort, UDP, SCTP. We can optimize this cluster by giving kube-proxy a way to not have to perform the duplicate work for these services.\n\nIt is important for overall scalability that kube-proxy does not receive data for Service/Endpoints objects that it is not going to affect. This can reduce load on the kube-proxy and the network by never receiving the updates in the first place.\n\nThe proposal is to make this feature available by annotating the Service object with this label: `service.kubernetes.io/service-proxy-name`. If this label key is set, with any value, the associated Endpoints object will automatically inherit that label from the Service object as well.\n\nWhen this label is set, kube-proxy will behave as if that service does not exist. None of the functionality that kube-proxy provides will be available for that service.\n\nkube-proxy will properly implement this label both at object creation and on dynamic addition/removal/updates of this label, either providing functionality or not for the service based on the latest version on the object.\n\nIt is optional for other service proxy implementations (besides kube-proxy) to implement this feature. They may ignore this value and still remain conformant with kubernetes services.\n\nIt is expected that this feature will mainly be used on large clusters with lots (\u003e1000) of services. Any use of this feature in a smaller cluster will have negligible impact.\n\nThe envisioned cluster that will make use of this feature looks something like the following:\n* Most/all traffic from outside the cluster is handled by gateways, such that each service in the cluster does not need a nodePort\n* These small number of entry points into the cluster are a part of the service mesh\n* There are many micro-services in the cluster, all a part of the service mesh, that are only accessed from inside the service mesh\n\nHigher level frameworks built on top of service meshes, such as [Knative](https://github.com/knative/docs), will be able to enable this feature by default due to having a more controlled application/service model and being reliant on the service mesh.\n\n#### Design\n\nCurrently, when ProxyServer starts up it creates informers for all Service (ServiceConfig) and Endpoints (EndpointsConfig) objects using a single shared informer factory.\n\nThe new design will simply add a LabelSelector filter to the shared informer factory, such that objects with the above label are filtered out by the API server:\n```diff\n-       informerFactory := informers.NewSharedInformerFactory(s.Client, s.ConfigSyncPeriod)\n+       informerFactory := informers.NewSharedInformerFactoryWithOptions(s.Client, s.ConfigSyncPeriod,\n+               informers.WithTweakListOptions(func(options *v1meta.ListOptions) {\n+                       options.LabelSelector = \"!service.kubernetes.io/service-proxy-name\"\n+               }))\n```\n\nThis code will also handle the dynamic label update case. When the label selector is matched (service is enabled) an 'add' event will be generated by the informer. When the label selector is not matched (service is disabled) a 'delete' event will be generated by the informer.\n\n#### Testing\n\nThe following cases should be tested. In each case, make sure that services are added/removed from iptables (or other) as expected:\n* Adding/removing services/endpoints with and without the above label\n* Adding/removing the above label from existing services/endpoints\n\n### Risks and Mitigations\n\nWe will keep the existing behaviour enabled by default, and only disable the kube-proxy service proxy when the service contains this new label.\n\nThis will have no effect on alternate service proxy implementations since they will not handle this label.\n\n## Graduation Criteria\n\nN/A\n\n## Implementation History\n\n- 2018-10-17 - This KEP is created\n- 2018-11-12 - KEP updated, including approver/reviewer\n"
+  },
+  {
+    "id": "e8b0dd7c703095f3cd2dcbb6e4bbfba4",
+    "title": "Kubernetes Dual-stack Support",
+    "authors": ["leblancd@", "rpothier@", "lachie83@", "khenidak@", "feiskyer@"],
+    "owningSig": "sig-network",
+    "participatingSigs": ["sig-cluster-lifecycle"],
+    "reviewers": ["TBD"],
+    "approvers": ["thockin@"],
+    "editor": "TBD",
+    "creationDate": "2018-05-21",
+    "lastUpdated": "2019-10-15",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# IPv4/IPv6 Dual-stack\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Plan](#implementation-plan)\n  - [Awareness of Multiple IPs per Pod](#awareness-of-multiple-ips-per-pod)\n  - [Required changes to Container Runtime Interface (CRI)](#required-changes-to-container-runtime-interface-cri)\n    - [Versioned API Change: PodStatus v1 core](#versioned-api-change-podstatus-v1-core)\n      - [Default Pod IP Selection](#default-pod-ip-selection)\n    - [PodStatus Internal Representation](#podstatus-internal-representation)\n    - [Maintaining Compatible Interworking between Old and New Clients](#maintaining-compatible-interworking-between-old-and-new-clients)\n      - [V1 to Core (Internal) Conversion](#v1-to-core-internal-conversion)\n      - [Core (Internal) to V1 Conversion](#core-internal-to-v1-conversion)\n  - [Awareness of Multiple NodeCIDRs per Node](#awareness-of-multiple-nodecidrs-per-node)\n    - [kubelet Startup Configuration for Dual-Stack Pod CIDRs](#kubelet-startup-configuration-for-dual-stack-pod-cidrs)\n    - [kube-proxy Startup Configuration for Dual-Stack Pod CIDRs](#kube-proxy-startup-configuration-for-dual-stack-pod-cidrs)\n    - ['kubectl get pods -o wide' Command Display for Dual-Stack Pod Addresses](#kubectl-get-pods--o-wide-command-display-for-dual-stack-pod-addresses)\n    - ['kubectl describe pod ...' Command Display for Dual-Stack Pod Addresses](#kubectl-describe-pod--command-display-for-dual-stack-pod-addresses)\n  - [Container Networking Interface (CNI) Plugin Considerations](#container-networking-interface-cni-plugin-considerations)\n  - [Services](#services)\n  - [Endpoints](#endpoints)\n  - [kube-proxy Operation](#kube-proxy-operation)\n    - [Kube-Proxy Startup Configuration Changes](#kube-proxy-startup-configuration-changes)\n      - [Multiple bind addresses configuration](#multiple-bind-addresses-configuration)\n      - [Multiple cluster CIDRs configuration](#multiple-cluster-cidrs-configuration)\n  - [CoreDNS Operation](#coredns-operation)\n  - [Ingress Controller Operation](#ingress-controller-operation)\n    - [GCE Ingress Controller: Out-of-Scope, Testing Deferred For Now](#gce-ingress-controller-out-of-scope-testing-deferred-for-now)\n  - [Load Balancer Operation](#load-balancer-operation)\n    - [Type ClusterIP](#type-clusterip)\n    - [Type NodePort](#type-nodeport)\n    - [Type Load Balancer](#type-load-balancer)\n  - [Cloud Provider Plugins Considerations](#cloud-provider-plugins-considerations)\n    - [Multiple bind addresses configuration](#multiple-bind-addresses-configuration-1)\n    - [Multiple cluster CIDRs configuration](#multiple-cluster-cidrs-configuration-1)\n  - [Container Environment Variables](#container-environment-variables)\n  - [Kubeadm Support](#kubeadm-support)\n    - [Kubeadm Configuration Options](#kubeadm-configuration-options)\n    - [Kubeadm-Generated Manifests](#kubeadm-generated-manifests)\n  - [vendor/github.com/spf13/pflag](#vendorgithubcomspf13pflag)\n  - [End-to-End Test Support](#end-to-end-test-support)\n  - [User Stories](#user-stories)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n  - [Dual-stack at the Edge](#dual-stack-at-the-edge)\n  - [Variation: Dual-Stack Service CIDRs (a.k.a. Full Dual-stack)](#variation-dual-stack-service-cidrs-aka-full-dual-stack)\n    - [Benefits](#benefits)\n    - [Changes Required](#changes-required)\n- [Test Plan](#test-plan)\n- [Graduation Criteria](#graduation-criteria)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis proposal adds IPv4/IPv6 dual-stack functionality to Kubernetes clusters. This includes the following concepts:\n- Awareness of multiple IPv4/IPv6 address assignments per pod\n- Native IPv4-to-IPv4 in parallel with IPv6-to-IPv6 communications to, from, and within a cluster\n\n## Motivation\n\nThe adoption of IPv6 has increased in recent years, and customers are requesting IPv6 support in Kubernetes clusters. To this end, the support of IPv6-only clusters was added as an alpha feature in Kubernetes Version 1.9. Clusters can now be run in either IPv4-only, IPv6-only, or in a \"single-pod-IP-aware\" dual-stack configuration. This \"single-pod-IP-aware\" dual-stack support is limited by the following restrictions:\n- Some CNI network plugins are capable of assigning dual-stack addresses on a pod, but Kubernetes is aware of only one address per pod.\n- Kubernetes system pods (api server, controller manager, etc.) can have only one IP address per pod, and system pod addresses are either all IPv4 or all IPv6.\n- Endpoints for services are either all IPv4 or all IPv6 within a cluster.\n- Service IPs are either all IPv4 or all IPv6 within a cluster.\n\nFor scenarios that require legacy IPv4-only clients or services (either internal or external to the cluster), the above restrictions mean that complex and expensive IPv4/IPv6 transition mechanisms (e.g. NAT64/DNS64, stateless NAT46, or SIIT/MAP) will need to be implemented in the data center networking.\n\nOne alternative to adding transition mechanisms would be to modify Kubernetes to provide support for IPv4 and IPv6 communications in parallel, for both pods and services, throughout the cluster (a.k.a. \"full\" dual-stack).\n\nA second, simpler alternative, which is a variation to the \"full\" dual-stack model, would be to provide dual-stack addresses for pods and nodes, but restrict service IPs to be single-family (i.e. allocated from a single service CIDR). In this case, service IPs in a cluster would be either all IPv4 or all IPv6, as they are now. Compared to a full dual-stack approach, this \"dual-stack pods / single-family services\" approach saves on implementation complexity, but would introduce some minor feature restrictions. (For more details on these tradeoffs, please refer to the \"Variation: Dual-Stack Service CIDRs\" section under \"Alternatives\" below).\n\nThis proposal aims to add \"dual-stack pods / single-family services\" support to Kubernetes clusters, providing native IPv4-to-IPv4 communication and native IPv6-to-IPv6 communication to, from and within a Kubernetes cluster.\n\n### Goals\n\n- Pod Connectivity: IPv4-to-IPv4 and IPv6-to-IPv6 access between pods\n- Access to External Servers: IPv4-to-IPv4 and IPv6-to-IPv6 access from pods to external servers\n- NGINX Ingress Controller Access: Access from IPv4 and/or IPv6 external clients to Kubernetes services via the Kubernetes NGINX Ingress Controller.\n- Dual-stack support for Kubernetes service NodePorts and ExternalIPs\n- Functionality tested with the Bridge CNI plugin, PTP CNI plugin, and Host-Local IPAM plugins as references\n- Maintain backwards-compatible support for IPv4-only and IPv6-only clusters\n\n### Non-Goals\n\n- Service CIDRs: Dual-stack service CIDRs will not be supported for this proposal. Service access within a cluster will be done via all IPv4 service IPs or all IPv6 service IPs.\n- Single-Family Applications: There may be some some clients or applications that only work with (bind to) IPv4 or or only work with (bind to) IPv6. A cluster can support either IPv4-only applications or IPv6-only applications (not both), depending upon the cluster CIDR's IP family. For example, if a cluster uses an IPv6 service CIDR, then IPv6-only applications will work fine, but IPv4-only applications in that cluster will not have IPv4 service IPs (and corresponding DNS A records) with which to access Kubernetes services. If a cluster needs to support legacy IPv4-only applications, but not IPv6-only applications, then the cluster should be configured with an IPv4 service CIDR.\n- Cross-family connectivity: IPv4-to-IPv6 and IPv6-to-IPv4 connectivity is considered outside of the scope of this proposal. (As a possible future enhancement, the Kubernetes NGINX ingress controller could be modified to load balance to both IPv4 and IPv6 addresses for each endpoint. With such a change, it's possible that an external IPv4 client could access a Kubernetes service via an IPv6 pod address, and vice versa).\n- CNI network plugins: Some plugins other than the Bridge, PTP, and Host-Local IPAM plugins may support Kubernetes dual-stack, but the development and testing of dual-stack support for these other plugins is considered outside of the scope of this proposal.\n- Multiple IPs vs. Dual-Stack: Code changes will be done in a way to facilitate future expansion to more general multiple-IPs-per-pod and multiple-IPs-per-node support. However, this initial release will impose \"dual-stack-centric\" IP address limits as follows:\n  - Pod addresses: 1 IPv4 address and 1 IPv6 addresses per pod maximum\n  - Node addresses: 1 IPv4 address and 1 IPv6 addresses per node maximum\n  - Service addresses: 1 service IP address per service\n- Dual-stack service discovery testing will be performed using coreDNS.\n- External load balancers that rely on Kubernetes services for load balancing functionality will only work with the IP family that matches the IP family of the cluster's service CIDR.\n- Dual-stack support for Kubernetes orchestration tools other than kubeadm (e.g. miniKube, KubeSpray, etc.) are considered outside of the scope of this proposal. Communication about how to enable dual-stack functionality will be documented appropriately in-order so that aformentioned tools may choose to enable it for use.\n\n## Proposal\n\nIn order to support dual-stack in Kubernetes clusters, Kubernetes needs to have awareness of and support dual-stack addresses for pods and nodes. Here is a summary of the proposal (details follow in subsequent sections):\n\n- Kubernetes needs to be made aware of multiple IPs per pod (limited to one IPv4 and one IPv6 address per pod maximum).\n- Link Local Addresses (LLAs) on a pod will remain implicit (Kubernetes will not display nor track these addresses).\n- Service Cluster IP Range `--service-cluster-ip-range=` will support the configuration of one IPv4 and one IPV6 address block) \n- Service IPs will be allocated from only a single family (either IPv4 or IPv6 as specified in the Service `spec.ipFamily` OR the first configured address block defined via `--service-cluster-ip-range=`).\n- Backend pods for a service can be dual-stack.\n- Endpoints addresses will match the address family of the Service IP address family (eg. An IPv6 Service IP will only have IPv6 Endpoints)\n- Kube-proxy iptables mode needs to drive iptables and ip6tables in parallel. This is required, even though service IP support is single-family, so that Kubernetes services can be exposed to clients external to the cluster via both IPv4 and IPv6. Support includes:\n  - Service IPs: IPv4 and IPv6 family support\n  - NodePort: Support listening on both IPv4 and IPv6 addresses\n  - ExternalIPs: Can be IPv4 or IPv6\n- Kube-proxy IPVS mode will support dual-stack functionality similar to kube-proxy iptables mode as described above. IPVS kube-router support for dual-stack, on the other hand, is considered outside of the scope of this proposal.\n- For health/liveness/readiness probe support, the default behavior will not change and an additional optional field would be added to the pod specification and is respected by kubelet. This will allow application developers to select a preferred IP family to use for implementing probes on dual-stack pods.\n- The pod status API changes will include a per-IP string map for arbitrary annotations, as a placeholder for future Kubernetes enhancements. This mapping is not required for this dual-stack design, but will allow future annotations, e.g. allowing a CNI network plugin to indicate to which network a given IP address applies. The appropriate hooks will be provided to enable CRI/CNI to provide these details.\n- Kubectl commands and output displays will need to be modified for dual-stack.\n- Kubeadm support will need to be added to enable spin-up of dual-stack clusters. Kubeadm support is required for implementing dual-stack continuous integration (CI) tests.\n- New e2e test cases will need to be added to test parallel IPv4/IPv6 connectivity between pods, nodes, and services.\n\n### Implementation Plan\n\nGiven the scope of this enhancement, it has been suggested that we break the implementation into discrete phases that may be spread out over the span of many release cycles. The phases are as follows:\n\nPhase 1 (Kubernetes 1.16)\n- API type modifications\n   - Kubernetes types\n   - CRI types\n- dual-stack pod networking (multi-IP pod)\n- kubenet multi-family support\n\nPhase 2 (Kubernetes 1.16)\n- Multi-family services including kube-proxy\n- Working with a CNI provider to enable dual-stack support\n- Change kubelet prober to support multi-address\n- Update component flags to support multiple `--bind-address`\n\nPhase 3 (Planned Kubernetes 1.17)\n- Kube-proxy iptables support for dual-stack\n- Downward API support for Pod.Status.PodIPs\n- Test e2e CNI plugin support for dual-stack\n- Pod hostfile support for IPv6 interface address\n- Kind support for dual-stack\n- e2e testing\n- EndpointSlice API support in kube-proxy for dual-stack (IPVS and iptables)\n- Dual-stack support for kubeadm\n- Expand container runtime support (containerd, CRI-O)\n\nPhase 4 and beyond\n- External dependencies, eg. cloud-provider, CNI, CRI, CoreDNS etc...\n\n### Awareness of Multiple IPs per Pod\n\nSince Kubernetes Version 1.9, Kubernetes users have had the capability to use dual-stack-capable CNI network plugins (e.g. Bridge + Host Local, Calico, etc.), using the \n[0.3.1 version of the CNI Networking Plugin API](https://github.com/containernetworking/cni/blob/spec-v0.3.1/SPEC.md), to configure multiple IPv4/IPv6 addresses on pods. However, Kubernetes currently captures and uses only IP address from the pod's main interface.\n\nThis proposal aims to extend the Kubernetes Pod Status and the Pod Network Status API so that Kubernetes can track and make use of one or many IPv4 addresses and one or many IPv6 address assignment per pod.\n\nCNI provides list of ips and their versions. Kubernetes currently just chooses to ignore this and use a single IP. This means this struct will need to follow the pod.Pod ip style of primary IP as is, and another slice of IPs, having Pod.IPs[0] == Pod.IP which will look like the following:\n\n    type PodNetworkStatus struct {\n      metav1.TypeMeta `json:\",inline\"`\n\n      // IP is the primary ipv4/ipv6 address of the pod. Among other things it is the address that -\n      //   - kube expects to be reachable across the cluster\n      //   - service endpoints are constructed with\n      //   - will be reported in the PodStatus.PodIP field (will override the IP reported by docker)\n      IP net.IP `json:\"ip\" description:\"Primary IP address of the pod\"`\n            IPs  []net.IP `json:\"ips\" description:\"list of ips\"`\n    }\n\nCNI as of today does not provide additional metadata to the IP. So this Properties field - speced in this KEP - will be empty until CNI spec includes properties. Although in theory we can copy the interface name, gateway etc. into this Properties map.\n\n### Required changes to Container Runtime Interface (CRI)\n\nThe PLEG loop + status manager of kubelet makes an extensive use of PodSandboxStatus call to wire up PodIP to api server, as a patch call to Pod Resources. The problem with this is the response message wraps PodSandboxNetworkStatus struct and this struct only carries one IP. This will require a change similar to the change described above. We will work with the CRI team to coordinate this change.\n\n    type PodSandboxNetworkStatus struct {\n      // IP address of the PodSandbox.\n      Ip string `protobuf:\"bytes,1,opt,name=ip,proto3\" json:\"ip,omitempty\"`\n            Ips []string `protobuf:\"bytes,1,opt,name=ip,proto3\" json:\"ip,omitempty\"`\n    }\n\n#### Versioned API Change: PodStatus v1 core\nIn order to maintain backwards compatibility for the core V1 API, this proposal retains the existing (singular) \"PodIP\" field in the core V1 version of the [PodStatus V1 core API](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#podstatus-v1-core), and adds a new array of structures that store pod IPs along with associated metadata for that IP. The metadata for each IP (refer to the \"Properties\" map below) will not be used by the dual-stack feature, but is added as a placeholder for future enhancements, e.g. to allow CNI network plugins to indicate to which physical network that an IP is associated. Retaining the existing \"PodIP\" field for backwards compatibility is in accordance with the [Kubernetes API change quidelines](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api_changes.md).\n```\n    // Default IP address allocated to the pod. Routable at least within the\n    // cluster. Empty if not yet allocated.\n    PodIP string `json:\"podIP,omitempty\" protobuf:\"bytes,6,opt,name=podIP\"`\n\n    // IP address information for entries in the (plural) PodIPs slice.\n    // Each entry includes:\n    //    IP: An IP address allocated to the pod. Routable at least within\n    //        the cluster.\n    //    Properties: Arbitrary metadata associated with the allocated IP.\n    type PodIPInfo struct {\n        IP string\n        Properties map[string]string\n    }\n\n    // IP addresses allocated to the pod with associated metadata. This list\n    // is inclusive, i.e. it includes the default IP address stored in the\n    // \"PodIP\" field, and this default IP address must be recorded in the\n    // 0th entry (PodIPs[0]) of the slice. The list is empty if no IPs have\n    // been allocated yet.\n    PodIPs []PodIPInfo `json:\"podIPs,omitempty\" protobuf:\"bytes,6,opt,name=podIPs\"`\n```\n\n##### Default Pod IP Selection\nOlder servers and clients that were built before the introduction of full dual-stack will only be aware of and make use of the original, singular PodIP field above. It is therefore considered to be the default IP address for the pod. When the PodIP and PodIPs fields are populated, the PodIPs[0] field must match the (default) PodIP entry. If a pod has both IPv4 and IPv6 addresses allocated, then the IP address chosen as the default IP address will match the IP family of the cluster's configured service CIDR. For example, if the service CIDR is IPv4, then the IPv4 address will be used as the default address.\n\n#### PodStatus Internal Representation\nThe PodStatus internal representation will be modified to use a slice of PodIPInfo structs rather than a singular IP (\"PodIP\"):\n```\n    // IP address information. Each entry includes:\n    //    IP: An IP address allocated to the pod. Routable at least within\n    //        the cluster.\n    //    Properties: Arbitrary metadata associated with the allocated IP.\n    // Empty if no IPs have been allocated yet.\n    type PodIPInfo struct {\n        IP string\n        Properties map[string]string\n    }\n\n    // IP addresses allocated to the pod with associated metadata.\n    PodIPs []PodIPInfo `json:\"podIPs,omitempty\" protobuf:\"bytes,6,opt,name=podIPs\"`\n```\nThis internal representation should eventually become part of a versioned API (after a period of deprecation for the singular \"PodIP\" field).\n\n#### Maintaining Compatible Interworking between Old and New Clients\nAny Kubernetes API change needs to consider consistent interworking between a possible mix of clients that are running old vs. new versions of the API. In this particular case, however, there is only ever one writer of the PodStatus object, and it is the API server itself. Therefore, the API server does not have an absolute requirement to implement any safeguards and/or fixups between the singular PodIP and the plural PodIPs fields as described in the guidelines for pluralizing singular API fields that is included in the [Kubernetes API change quidelines](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api_changes.md).\n\nHowever, as a defensive coding measure and for future-proofing, the following API version translation logic will be implemented for the PodIP/PodIPs fields:\n\n##### V1 to Core (Internal) Conversion\n- If only V1 PodIP is provided:\n  - Copy V1 PodIP to core PodIPs[0]\n- Else if only V1 PodIPs[] is provided: # Undetermined as to whether this can actually happen (@thockin)\n  - Copy V1 PodIPs[] to core PodIPs[]\n- Else if both V1 PodIP and V1 PodIPs[] are provided:\n  - Verify that V1 PodIP matches V1 PodIPs[0]\n  - Copy V1 PodIPs[] to core PodIPs[]\n- Delete any duplicates in core PodIPs[]\n\n##### Core (Internal) to V1 Conversion\n  - Copy core PodIPs[0] to V1 PodIP\n  - Copy core PodIPs[] to V1 PodIPs[]\n\n### Awareness of Multiple NodeCIDRs per Node\nAs with PodIP, corresponding changes will need to be made to NodeCIDR. These changes are essentially the same as the aformentioned PodIP changes which create the pularalization of NodeCIDRs to a slice rather than a singular and making those changes across the internal representation and v1 with associated conversations.\n\n\n#### kubelet Startup Configuration for Dual-Stack Pod CIDRs\nThe existing \"--pod-cidr\" option for the [kubelet startup configuration](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/) will be modified to support multiple IP CIDRs in a comma-separated list (rather than a single IP string), i.e.:\n```\n  --pod-cidr  ipNetSlice   [IP CIDRs, comma separated list of CIDRs, Default: []]\n```\nOnly the first address of each IP family will be used; all others will be logged and ignored.\n\n#### kube-proxy Startup Configuration for Dual-Stack Pod CIDRs\nThe existing \"cluster-cidr\" option for the [kube-proxy startup configuration](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/) will be modified to support multiple cluster CIDRs in a comma-separated list (rather than a single IP string), i.e:\n```\n  --cluster-cidr  ipNetSlice   [IP CIDRs, comma separated list of CIDRs, Default: []]\n```\nOnly the first address of each IP family will be used; all others will be logged and ignored.\n\n#### 'kubectl get pods -o wide' Command Display for Dual-Stack Pod Addresses\nThe output for the 'kubectl get pods -o wide' command will not need modification and will only display the primary pod IP address as determined by the first IP address block configured via the `--cluster-cidr=` on kube-controller-manager. eg. The following is expected output for a cluster is configured with an IPv4 address block as the first configured via the `--cluster-cidr=` on kube-controller-manager:\n```\n       kube-master# kubectl get pods -o wide\n       NAME               READY     STATUS    RESTARTS   AGE       IP                          NODE\n       nginx-controller   1/1       Running   0          20m       10.244.2.7                  n01\n       kube-master#\n```\n\nFor comparison, here expected output for a cluster is configured with an IPv6 address block as the first configured via the `--cluster-cidr=` on kube-controller-manager:\n```\n       kube-master# kubectl get pods -o wide\n       NAME               READY     STATUS    RESTARTS   AGE       IP                          NODE\n       nginx-controller   1/1       Running   0          20m       fd00:db8:1::2               n01\n       kube-master#\n```\n\n#### 'kubectl describe pod ...' Command Display for Dual-Stack Pod Addresses\nThe output for the 'kubectl describe pod ...' command will need to be modified to display a list of IPs for each pod, e.g.:\n```\n       kube-master# kubectl describe pod nginx-controller\n       .\n       .\n       .\n        IP:           10.244.2.7\n        IPs:\n          IP:           10.244.2.7\n          IP:           fd00:200::7\n       .\n       .\n       .\n```\n\n### Container Networking Interface (CNI) Plugin Considerations\n\nThis feature requires the use of the [CNI Networking Plugin API version 0.3.1](https://github.com/containernetworking/cni/blob/spec-v0.3.1/SPEC.md)\nor later. The dual-stack feature requires no changes to this API.\n\nThe versions of CNI plugin binaries that must be used for proper dual-stack functionality (and IPv6 functionality in general) depend upon the version of the container runtime that is used in the cluster nodes (see [CNI issue #531](https://github.com/containernetworking/cni/issues/531) and [CNI plugins PR #113](https://github.com/containernetworking/plugins/pull/113)):\n\n### Services\n\nServices will only be assigned an address from a single address family (either IPv4 or IPv6). The address family for the Service’s cluster IP is determined by setting the field, `.spec.ipFamily`, on that Service. You can only set this field when creating a new Service. Setting the `.spec.ipFamily` field is optional and should only be used if you plan to enable IPv4 and IPv6 Services and Ingresses on your cluster. The configuration of this field not a requirement for egress traffic. The default address family for your cluster is the address family of the first service cluster IP range configured via the `--service-cluster-ip-range` flag to the kube-controller-manager. The `.spec.ipFamily` field can be set to either IPv4 or IPv6.\n\nFor example, the following Service specification includes the `spec.ipFamily` field. Kubernetes will assign an IPv6 address (also known as a “cluster IP”) from the configured service-cluster-ip-range to this Service.\n\n```yaml\napiVersion: v1\nkind: Service\nmetadata:\n  name: my-service\nspec:\n  ipFamily: IPv6\n  selector:\n    app: MyApp\n  ports:\n    - protocol: TCP\n      port: 80\n      targetPort: 9376\n```\n\nDual-stack services (having both an IPv4 and IPv6 cluster IP) are currently out of scope of this proposal. Should there be use-cases where dual-stack Services are needed then we can revisit.\n\n### Endpoints\n\nThe current [Kubernetes Endpoints API](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#endpoints-v1-core) (i.e. before the addition of the dual-stack feature), supports only a single IP address per endpoint. With the addition of the dual-stack feature, pods serving as backends for Kubernetes services may now have both IPv4 and IPv6 addresses. This presents a design choice of how to represent such dual-stack endpoints in the Endpoints API. Two choices worth considering would be:\n- 2 single-family endpoints per backend pod: Make no change to the [Kubernetes Endpoints API](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#endpoints-v1-core). Treat each IPv4/IPv6 address as separate, distinct endpoints, and include each address in the comma-separated list of addresses in an 'Endpoints' API object. \n- 1 dual-stack endpoint per backend pod: Modify the [Kubernetes Endpoints API](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#endpoints-v1-core) so that each endpoint can be associated with a pair of IPv4/IPv6 addresses.\n\nGiven a phased approach, the 2 single-family endpoint approach represents the least disruptive change. Services will select only the endpoints that match the `spec.ipFamily` defined in the Service.\n\nFor example, for a Service named `my-service` that has the `spec.ipFamily` set to IPv4 would only have endpoints only from the IPv4 address family.\n\n```bash\n$ kubectl get ep my-service\nNAME         ENDPOINTS                                         AGE\nmy-service   10.244.0.6:9376,10.244.2.7:9376,10.244.2.8:9376   84s\n```\n\nLikewise, for a Service named `my-service-v6` that has the `spec.ipFamily` set to IPv6 would only have endpoints from the IPv6 address family.\n\n```bash\n$ kubectl get ep my-service-v6\nNAME            ENDPOINTS                                              AGE\nmy-service-v6   [fd00:200::7]:9376,[fd00:200::8]:9376,[fd00::6]:9376   3m28s\n```\n\n### kube-proxy Operation\n\nKube-proxy will be modified to drive iptables and ip6tables in parallel. This will require the implementation of a second \"proxier\" interface in the Kube-Proxy server in order to modify and track changes to both tables. This is required in order to allow exposing services via both IPv4 and IPv6, e.g. using Kubernetes:\n  - NodePort\n  - ExternalIPs\n\n#### Kube-Proxy Startup Configuration Changes\n\n##### Multiple bind addresses configuration\nThe existing \"--bind-address\" option for the [kube-proxy startup configuration](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/) will be modified to support multiple IP addresses in a comma-separated list (rather than a single IP string).\n```\n  --bind-address  stringSlice   (IP addresses, in a comma separated list, Default: [0.0.0.0,])\n```\nOnly the first address of each IP family will be used; all others will be ignored.\n\n##### Multiple cluster CIDRs configuration\nThe existing \"--cluster-cidr\" option for the [kube-proxy startup configuration](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/) will be modified to support multiple IP CIDRs in a comma-separated list (rather than a single IP CIDR).\nA new [kube-proxy configuration](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/) argument will be added to allow a user to specify multiple cluster CIDRs.\n```\n  --cluster-cidr  ipNetSlice   (IP CIDRs, in a comma separated list, Default: [])\n```\nOnly the first CIDR for each IP family will be used; all others will be ignored.\n\n### CoreDNS Operation\n\nCoreDNS will need to make changes in order to support the plural form of endpoint addresses. Some other considerations of CoreDNS support for dual-stack:\n\n- Because service IPs will remain single-family, pods will continue to access the CoreDNS server via a single service IP. In other words, the nameserver entries in a pod's /etc/resolv.conf will typically be a single IPv4 or single IPv6 address, depending upon the IP family of the cluster's service CIDR.\n- Non-headless Kubernetes services: CoreDNS will resolve these services to either an IPv4 entry (A record) or an IPv6 entry (AAAA record), depending upon the IP family of the cluster's service CIDR.\n- Headless Kubernetes services: CoreDNS will resolve these services to either an IPv4 entry (A record), an IPv6 entry (AAAA record), or both, depending on the service's `ipFamily`.\n\n### Ingress Controller Operation\n\nThe [Kubernetes ingress feature](https://kubernetes.io/docs/concepts/services-networking/ingress/) relies on the use of an ingress controller. The two \"reference\" ingress controllers that are considered here are the [GCE ingress controller](https://github.com/kubernetes/ingress-gce/blob/master/README.md#glbc) and the [NGINX ingress controller](https://github.com/kubernetes/ingress-nginx/blob/master/README.md#nginx-ingress-controller).\n\n#### GCE Ingress Controller: Out-of-Scope, Testing Deferred For Now\nIt is not clear whether the [GCE ingress controller](https://github.com/kubernetes/ingress-gce/blob/master/README.md#glbc) supports external, dual-stack access. Testing of dual-stack access to Kubernetes services via a GCE ingress controller is considered out-of-scope until after the initial implementation of dual-stack support for Kubernetes.\n\n#### NGINX Ingress Controller - Dual-Stack Support for Bare Metal Clusters\nThe [NGINX ingress controller](https://github.com/kubernetes/ingress-nginx/blob/master/README.md#nginx-ingress-controller) should provide dual-stack external access to Kubernetes services that are hosted on baremetal clusters, with little or no changes.\n\n- Dual-stack external access to NGINX ingress controllers is not supported with GCE/GKE or AWS cloud platforms.\n- NGINX ingress controller needs to be run on a pod with dual-stack external access.\n- On the load balancer (internal) side of the NGINX ingress controller, the controller will load balance to backend service pods on a per dual-stack-endpoint basis, rather than load balancing on a per-address basis. For example, if a given backend pod has both an IPv4 and an IPv6 address, the ingress controller will treat the IPv4 and IPv6 address endpoints as a single load-balance target. Support of dual-stack endpoints may require upstream changes to the NGINX ingress controller.\n- Ingress access can cross IP families. For example, an incoming L7 request that is received via IPv4 can be load balanced to an IPv6 endpoint address in the cluster, and vice versa. \n\n### Load Balancer Operation\n\nAs noted above, External load balancers that rely on Kubernetes services for load balancing functionality will only work with the IP family that matches the IP family of the cluster's service CIDR.\n\n#### Type ClusterIP\n\nThe ClusterIP service type will be single stack, so for this case there will be no changes to the current load balancer config. The user has the option to create two load balancer IP resources, one for IPv6 and the other for IPv4, and associate both with the same application instances.\n\n#### Type NodePort\n\nThe NodePort Service type uses the nodes IP address, which can be dual-stack, and port. If the Service type is NodePort, the `spec.ipFamily` will be used to determine how the the load balancer is configured and only the corresponding address family will be forwarded.\n\n#### Type Load Balancer\n\nThe cloud provider will provision an external load balancer. If the cloud provider load balancer maps directly to the pod iP's then a dual-stack load balancer could be used. Additional information may need to be provided to the cloud provider to configure dual-stack. See Implementation Plan for further details.\n\n### Cloud Provider Plugins Considerations\n\nThe [Cloud Providers](https://kubernetes.io/docs/concepts/cluster-administration/cloud-providers/) may have individual requirements for dual-stack in addition to below.\n\n#### Multiple bind addresses configuration\n\nThe existing \"--bind-address\" option for the will be modified to support multiple IP addresses in a comma-separated list (rather than a single IP string).\n```\n  --bind-address  stringSlice   (IP addresses, in a comma separated list, Default: [0.0.0.0,])\n```\nOnly the first address of each IP family will be used; all others will be ignored.\n\n#### Multiple cluster CIDRs configuration\n\nThe existing \"--cluster-cidr\" option for the [cloud-controller-manager](https://kubernetes.io/docs/reference/command-line-tools-reference/cloud-controller-manager/) will be modified to support multiple IP CIDRs in a comma-separated list (rather than a single IP CIDR).\n```\n  --cluster-cidr  ipNetSlice   (IP CIDRs, in a comma separated list, Default: [])\n```\nOnly the first CIDR for each IP family will be used; all others will be ignored.\n\nThe cloud_cidr_allocator will be updated to support allocating from multiple CIDRs. The route_controller will be updated to create routes for multiple CIDRs.\n\n### Container Environment Variables\n\nThe [container environmental variables](https://kubernetes.io/docs/concepts/containers/container-environment-variables/#container-environment) should support dual-stack.\n\nPod information is exposed through environmental variables on the pod. There are a few environmental variables that are automatically created, and some need to be specified in the pod definition, through the downward api.\n\nThe Downward API [status.podIP](https://kubernetes.io/docs/tasks/inject-data-application/downward-api-volume-expose-pod-information/#capabilities-of-the-downward-api) will preserve the existing single IP address, and will be set to the default IP for each pod. A new environmental variable named status.podIPs will contain a space-separated list of IP addresses. The new pod API will have a slice of structures for the additional IP addresses. Kubelet will translate the pod structures and return podIPs as a comma-delimited string.\n\nHere is an example of how to define a pluralized MY_POD_IPS environmental variable in a pod definition yaml file:\n```\n  - name: MY_POD_IPS\n    valueFrom:\n      fieldRef:\n        fieldPath: status.podIPs\n```\n\nThis definition will cause an environmental variable setting in the pod similar to the following:\n```\nMY_POD_IPS=fd00:10:20:0:3::3,10.20.3.3\n```\n\n### Kubeadm Support\n\nDual-stack support will need to be added to kubeadm both for dual-stack development purposes, and for use in dual-stack continuous integration tests.\n\n- The Kubeadm config options and config file will support dual-stack options for apiserver-advertise-address, and podSubnet.\n\n#### Kubeadm Configuration Options\n\nThe kubeadm configuration options for advertiseAddress and podSubnet will need to be changed to handle a comma-separated list of CIDRs:\n```\n    api:\n      advertiseAddress: \"fd00:90::2,10.90.0.2\" [Multiple IP CIDRs, comma separated list of CIDRs]\n    networking:\n      podSubnet: \"fd00:10:20::/72,10.20.0.0/16\" [Multiple IP CIDRs, comma separated list of CIDRs]\n```\n\n#### Kubeadm-Generated Manifests\n\nKubeadm will need to generate dual-stack CIDRs for the --service-cluster-ip-range command line argument in kube-apiserver.yaml:\n```\n    spec:\n      containers:\n      - command:\n        - kube-apiserver\n        - --service-cluster-ip-range=fd00:1234::/110,10.96.0.0/12\n```\n\nKubeadm will also need to generate dual-stack CIDRs for the --cluster-cidr argument in kube-apiserver.yaml:\n```\n    spec:\n      containers:\n      - command:\n        - kube-controller-manager\n        - --cluster-cidr=fd00:10:20::/72,10.20.0.0/16\n```\n\n### vendor/github.com/spf13/pflag\nThis dual-stack proposal will introduce a new IPNetSlice object to spf13.pflag to allow parsing of comma separated CIDRs. Refer to [https://github.com/spf13/pflag/pull/170](https://github.com/spf13/pflag/pull/170)\n\n### End-to-End Test Support\nEnd-to-End tests will be updated for dual-stack. The dual-stack e2e tests will utilized kubernetes-sigs/kind along with supported cloud providers. The e2e test-suite for dual-stack is running [here](https://testgrid.k8s.io/sig-network-dualstack-azure-e2e#dualstack-azure-e2e). Once dual-stack support is added to kind, corresponding dual-stack e2e tests will be run on kind similar to [this](https://testgrid.k8s.io/sig-release-master-blocking#kind-ipv6-master-parallel). \n\nThe E2E test suite that will be run for dual-stack will be based upon the [IPv6-only test suite](https://github.com/CiscoSystems/kube-v6-test) as a baseline. New versions of the network connectivity test cases that are listed below will need to be created so that both IPv4 and IPv6 connectivity to and from a pod can be tested within the same test case. A new dual-stack test flag will be created to control when the dual-stack tests are run versus single stack versions of the tests:\n```\n[It] should function for node-pod communication: udp [Conformance]\n[It] should function for node-pod communication: http [Conformance]\n[It] should function for intra-pod communication: http [Conformance]\n[It] should function for intra-pod communication: udp [Conformance]\n```\nMost service test cases do not need to be updated as the service remains single stack.\n\nFor the test that checks pod internet connectivity, the IPv4 and IPv6 tests can be run individually, with the same initial configurations.\n```\n[It] should provide Internet connection for containers\n```\n\n### User Stories\n\\\u003cTBD\\\u003e\n\n### Risks and Mitigations\n\\\u003cTBD\\\u003e\n\n\n## Implementation History\nRefer to the [Implementation Plan](#implementation-plan)\n\n## Alternatives\n\n### Dual-stack at the Edge\nInstead of modifying Kubernetes to provide dual-stack functionality within the cluster, one alternative is to run a cluster in IPv6-only mode, and instantiate IPv4-to-IPv6 translation mechanisms at the edge of the cluster. Such an approach can be called \"Dual-stack at the Edge\". Since the translation mechanisms are mostly external to the cluster, very little changes (or integration) would be required to the Kubernetes cluster itself. (This may be quicker for Kubernetes users to implement than waiting for the changes proposed in this proposal to be implemented).\n\nFor example, a cluster administrator could configure a Kubernetes cluster in IPv6-only mode, and then instantiate the following external to the cluster:\n- Stateful NAT64 and DNS64 servers: These would handle connections from IPv6 pods to external IPv4-only servers. The NAT64/DNS64 servers would be in the data center, but functionally external to the cluster. (Although one variation to consider would be to implement the DNS64 server inside the cluster as a CoreDNS plugin.)\n- Dual-stack ingress controllers (e.g. Nginx): The ingress controller would need dual-stack access on the external side, but would load balance to IPv6-only endpoints inside the cluster.\n- Stateless NAT46 servers: For access from IPv4-only, external clients to Kubernetes pods, or to exposed services (e.g. via NodePort or ExternalIPs). This may require some static configuration for IPv4-to-IPv6 mappings.\n\n### Variation: Dual-Stack Service CIDRs (a.k.a. Full Dual-stack)\n\nAs a variation to the \"Dual-Stack Pods / Single-Family Services\" approach outlined above, we can consider supporting IPv4 and IPv6 service CIDRs in parallel (a.k.a. the \"full\" dual-stack approach).\n\n#### Benefits\nProviding dual-stack service CIDRs would add the following functionality:\n- Dual-Stack Pod-to-Services. Clients would have a choice of using A or AAAA DNS records when resolving Kubernetes services.\n- Simultaneous support for both IPv4-only and IPv6-only applications internal to the cluster. Without dual-stack service CIDRs, a cluster can support either IPv4-only applications or IPv6-only applications, depending upon the cluster CIDR's IP family. For example, if a cluster uses an IPv6 service CIDR, then IPv4-only applications in that cluster will not have IPv4 service IPs (and corresponding DNS A records) with which to access Kubernetes services.\n- External load balancers that use Kubernetes services for load balancing functionality (i.e. by mapping to service IPs) would work in dual-stack mode. (Without dual-stack service CIDRs, these external load balancers would only work for the IP family that matches the cluster service CIDR's family.)\n\n#### Changes Required\n- [controller-manager startup configuration](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/): The \"--service-cluster-ip-range\" startup argument would need to be modified to accept a comma-separated list of CIDRs.\n- [kube-apiserver startup configuration](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/): The \"--service-cluster-ip-range\" would need to be modified to accept a comma-separated list of CIDRs.\n- [Service V1 core API](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#service-v1-core): This versioned API object would need to be modified to support multiple cluster IPs for each service. This would require, for example, the addition of an \"ExtraClusterIPs\" slice of strings, and the designation of one of the cluster IPs as the default cluster IP for a given service (similar to changes described above for the PodStatus v1 core API).\n- The service allocator: This would need to be modified to allocate a service IP from each service CIDR for each service that is created.\n- 'kubectl get service' command: The display output for this command would need to be modified to return multiple service IPs for each service.\n- CoreDNS may need to be modified to loop through both (IPv4 and IPv6) service IPs for each given Kubernetes service, and advertise both IPs as A and AAAA records accordingly in DNS responses.\n\n## Test Plan\n\n* Test-grid e2e tests - https://testgrid.k8s.io/sig-network-dualstack-azure-e2e\n* e2e tests\n  * [sig-network] [Feature:IPv6DualStackAlphaFeature] [LinuxOnly] should create pod, add ipv6 and ipv4 ip to pod ips\n  * [sig-network] [Feature:IPv6DualStackAlphaFeature] [LinuxOnly] should have ipv4 and ipv6 internal node ip\n  * [sig-network] [Feature:IPv6DualStackAlphaFeature] [LinuxOnly] should have ipv4 and ipv6 node podCIDRs\n  * [sig-network] [Feature:IPv6DualStackAlphaFeature] [LinuxOnly] should be able to reach pod on ipv4 and ipv6 ip [Feature:IPv6DualStackAlphaFeature:Phase2]\n  * [sig-network] [Feature:IPv6DualStackAlphaFeature] [LinuxOnly] should create service with cluster ip from primary service range [Feature:IPv6DualStackAlphaFeature:Phase2]\n  * [sig-network] [Feature:IPv6DualStackAlphaFeature] [LinuxOnly] should create service with ipv4 cluster ip [Feature:IPv6DualStackAlphaFeature:Phase2]\n  * [sig-network] [Feature:IPv6DualStackAlphaFeature] [LinuxOnly] should create service with ipv6 cluster ip [Feature:IPv6DualStackAlphaFeature:Phase2]\n* e2e tests should cover the following:\n  * multi-IP, same family\n  * multi-IP, dual-stack\n  * single IP, ipv4\n  * single IP, ipv6\n## Graduation Criteria\n\nThis capability will move to beta when the following criteria have been met.\n\n* Kubernetes types finalized\n* CRI types finalized\n* Pods to support multi-IPs\n* Nodes to support multi-CIDRs\n* Service resource supports pods with multi-IP\n* Kubenet to support multi-IPs\n\nThis capability will move to stable when the following criteria have been met.\n\n* Support of at least one CNI plugin to provide multi-IP\n* e2e test successfully running on two platforms\n* testing ingress controller infrastructure with updated dual-stack services\n"
+  },
+  {
+    "id": "f524dd4bb6da3b51bc3cd06aafa57467",
+    "title": "Topology-aware service routing",
+    "authors": ["@m1093782566", "@andrewsykim"],
+    "owningSig": "sig-network",
+    "participatingSigs": null,
+    "reviewers": ["@thockin", "@johnbelamaric"],
+    "approvers": ["@thockin"],
+    "editor": "",
+    "creationDate": "2018-10-24",
+    "lastUpdated": "2019-10-17",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Topology-aware service routing\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-goals](#non-goals)\n  - [User cases](#user-cases)\n  - [Background](#background)\n- [Proposal](#proposal)\n- [Design Details](#design-details)\n  - [Service API changes](#service-api-changes)\n  - [Intersection with Local External Traffic Policy](#intersection-with-local-external-traffic-policy)\n  - [Service Topology Scalability](#service-topology-scalability)\n    - [New PodLocator resource](#new-podlocator-resource)\n    - [New PodLocator controller](#new-podlocator-controller)\n  - [Kube-proxy changes](#kube-proxy-changes)\n  - [DNS server changes (in beta stage)](#dns-server-changes-in-beta-stage)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n    - [Alpha](#alpha)\n\u003c!-- /toc --\u003e\n\n## Motivation\n\nFigure out a generic way to implement the \"local service\" route, say \"topology aware routing of service\".\n\nLocality is defined by user, it can be any topology-related thing. \"Local\" means the \"same topology level\", e.g. same node, same rack, same failure zone, same failure region, same cloud provider etc. Two nodes are considered \"local\" if they have the same value for a particular label, called the \"topology key\".\n\n### Goals\n\nA generic way to support topology aware routing of services in arbitrary topological domains, e.g. node, rack, zone, region, etc. by node labels.\n\n### Non-goals\n\n* Scheduler spreading to implement this sort of topology guarantee\n* Dynamic Availability\n* Health-checking\n* Capacity-based or load-based spillover\n\n### User cases\n\n* Logging agents such as fluentd. Deploy fluentd as DaemonSet and applications only need to communicate with the fluentd in the same node.\n* For a sharded service that keeps per-node local information in each shard.\n* Authenticating proxies such as [aws-es-proxy](https://github.com/kopeio/aws-es-proxy).\n* In container identity wg, being able to give daemonset pods a unique identity per host is on the 2018 plan, and ensuring local pods can communicate to local node services securely is a key goal there. -- from @smarterclayton\n* Regional data costs in multi-AZ setup - for instance, in AWS, with a multi-AZ setup, half of the traffic will switch AZ, incurring regional data Transfer costs, whereas if something was local, it wouldn't hit the network.\n* Performance benefit (node local/rack local) is lower latency/higher bandwidth.\n\n### Background\n\nIt's a pain point for multi-zone clusters deployment since cross-zone network traffic being charged, while in-zone is not. In addition, cross-node traffic may carry sensitive metadata from other nodes. Therefore, users always prefer the service backends that close to them, e.g. same zone, rack and host etc. for security, performance and cost concerns.\n\nKubernetes scheduler can constraining a pod to only be able to run on particular nodes/zones. However, Kubernetes service proxy just randomly picks an available backend for service routing and this one can be very far from the user, so we need a topology-aware service routing solution in Kubernetes. Basically, to find the nearest service backend. In other words, allowing people to configure if ALWAY reach a to local service backend. In this way, they can reduce network latency, improve security, save money and so on. However, because topology is arbitrary, zone, region, rack, generator, whatever, who knows? We should allow arbitrary locality.\n\n`ExternalTrafficPolicy` was added in v1.4, but only for NodePort and external LB traffic. NodeName was added to `EndpointAddress` to allow kube-proxy to filter local endpoints for various future purposes.\n\nBased on our experience of advanced routing setup and recent demo of enabling this feature in Kubernetes, this document would like to introduce a more generic way to support arbitrary service topology.\n\n## Proposal\n\nThis proposal builds off of earlier requests to [use local pods only for kube-proxy loadbalancing](https://github.com/kubernetes/kubernetes/issues/7433) and [node-local service proposal](https://github.com/kubernetes/kubernetes/pull/28637). But, this document proposes that not only the particular \"node-local\" user case should be taken care, but also a more generic way should be figured out.\n\nLocality is an \"user-defined\" thing. When we set topology key \"hostname\" for service, we expect node carries different node labels on the key \"hostname\".\n\nUsers can control the level of topology. For example, if someone run logging agent as a daemonset, they can set the \"hard\" topology requirement for same-host. If \"hard\" is not met, then just return \"service not available\".\n\nAnd if someone set a \"soft\" topology requirement for same-host, say they \"preferred\" same-host endpoints and can accept other hosts when for some reasons local service's backend is not available on some host.\n\nIf multiple endpoints satisfy the \"hard\" or \"soft\" topology requirement, we will randomly pick one by default.\n\nRouting decision is expected to be implemented by kube-proxy and kube-dns/coredns for headless service.\n\n\n## Design Details\n\n### Service API changes\n\nUsers need a way to declare what service is local and the definition of local backends for the particular service.\n\nIn this proposal, we give the service owner a chance to configure the service locality things. A new property would be introduced to `ServiceSpec`, say `topologyKeys` - it's a string slice and should be optional.\n\n```go\ntype ServiceSpec struct {\n  // topologyKeys is a preference-order list of topology keys.  If backends exist for\n  // index [0], they will always be chosen; only if no backends exist for index [0] will backends for index [1] be considered.\n  // If this field is specified and all indices have no backends, the service has no backends, and connections will fail.  We say these requirements are hard.\n  // In order to express soft requirement, we may give a special node label key \"*\" as it means \"match all nodes\".\n  TopologyKeys []string `json:\"topologyKeys\" protobuf:\"bytes,1,opt,name=topologyKeys\"`\n}\n```\n\nAn example of `Service` with topology keys:\n\n```\nkind: Service\nmetadata:\n  name: service-local\nspec:\n  topologyKeys: [\"kubernetes.io/hostname\", \"topology.kubernetes.io/zone\"]\n```\n\n\nIn our example above, we will firstly try to find the backends in the same host. If no backends match, we will then try the same zone. If finally we can't find any backends in the same host or same zone, then we say the service has no satisfied backends and connections will fail.\n\nIf we configure topologyKeys as `[\"kubernetes.io/hostname\", \"*\"]`, we just do the effort to find the backends in the same host and will not fail the connection if no matched backends found.\n\n### Intersection with Local External Traffic Policy\n\nTopology Aware Services (# of topologyKeys \u003e 0) and Services using `externalTrafficPolicy=Local` will be mutually exclusive. This will be enforced by API validation where a Service with `externalTrafficPolicy=Local` cannot specify any topology keys and vice versa.\n\nBefore Service Topology goes GA, kube-proxy should be able to provide feature parity with `externalTrafficPolicy=Local` when the chosen topology is `kubernetes.io/hostname`. The correctness and feasibility of the `kubernetes.io/hostname` topology as a sufficient replacement for `externalTrafficPolicy=Local` will be evaluated during the alpha phase.\n\n### Service Topology Scalability\n\nThe alpha release of Service Topology will have a simple implementation where kube-proxy and any other consumers of `topologyKeys` will determine the locality of backends\nby filtering Endpoints and Nodes in-memory from client-go's cache informer/lister. The alpha implementation of Service Topology will not account for scalabilty, however,\nthe beta release of Service Topology will address scalability concerns by introducing a `PodLocator` resource. Note that the design and implementation details around `PodLocator`\nmay vary based on user feedback in the alpha release.\n\n#### New PodLocator resource\n\nAs `EndpointAddress` already contains the `nodeName` field, we can build a service that will preemptively map Pods to their topologies. From there, all interested components (at least kube-proxy, kube-dns, and coredns) can watch the `PodLocator` object and do necessary mappings internally. Given that we don't know which labels are topology labels, we are going to copy all node labels to `PodLocator`.\n\n```\n// PodLocator represents information about where a pod exists in arbitrary space.  This is useful for things like\n// being able to reverse-map pod IPs to topology labels, without needing to watch all Pods or all Nodes.\ntype PodLocator struct {\n    metav1.TypeMeta\n    // +optional\n    metav1.ObjectMeta\n\n    // NOTE: Fields in this resource must be relatively small and relatively low-churn.\n\n    IPs []PodIPInfo // being added for dual-stack support\n    NodeName string\n    NodeLabels map[string]string\n}\n```\n\nIn order to reference PodLocator back to a Pod easily, PodLocator namespace and name would be 1:1 with Pod namespace and name. In other words, PodLocator is a lightweight object which stores Pod location/topology information.\n\n**NOTE**: whether `PodLocator` will be a core API resource or a CRD will be determined after the alpha release of Service Topology.\n\n#### New PodLocator controller\n\nA new PodLocator controller will watch and cache all Pods and Nodes. Then pre-cook pod name to {pod IPs, node name, node labels} mapping.\n\nWhen a Pod is added, PodLocator controller will created a new PodLocator object whose namespace and name are 1:1 with Pod namespace and name. Then it will populate the Pod's IP(s), node name and labels into the new object.\n\nWhen a Pod is updated, PodLocator controller will first check if IPs or Spec.NodeName are changed. If changed, PodLocator controller will update the corresponding PodLocator object accordingly, otherwise will ignore this change.\n\nWhen a Pod is deleted, PodLocator controller will delete the corresponding PodLocator object.\n\nWhen a Node is updated, PodLocator controller will first check if its labels are changed. If changed, will update all the PodLocators whose corresponding Pods running on it.\n\nWhen a Node is deleted, PodLocator controller will reset the NodeName and NodeLabels of all the PodLocators whose corresponding Pods running on it.\n\n### Kube-proxy changes\n\nKube-proxy will respect topology keys for each service, so kube-proxy on different nodes may create different proxy rules.\n\nKube-proxy will watch its own node and will find the endpoints that are in the same topological domain as the node if `service.TopologyKeys` is not empty.\n\nFor the beta release, kube-proxy will watch the PodLocator apart from Service and Endpoints. For each Endpoints object, kube-proxy will find the original Pod via EndpointAddress.TargetRef, therefore will get PodLocator object and its topology information. Kube-proxy will only create proxy rules for endpoints that are in the same topological domain as the node running kube-proxy.\n\n### DNS server changes (in beta stage)\n\nWe should consider this kind of topology support for headless service in coredns and kube-dns. As the DNS servers will respect topology keys for each headless service, different clients/pods on different nodes may get different dns response.\n\nIn order to handle headless services, the DNS server needs to know the node corresponding to the client IP address in the DNS request - i.e, it needs to map PodIP -\u003e Node. Kubernetes DNS servers(include kube-dns and CoreDNS) will watch PodLocator object. When a client/pod request a headless service domain to DNS server, dns server will retrieve the node labels of both client and the backend Pods via PodLocator. DNS server will only select the IPs of backend Pods which are in the same topological domain with client Pod, and then write A record.\n\n### Test Plan\n\n* Unit tests to check API validation of topology keys.\n* Unit tests to check API validation of incompatibility with\n  externalTrafficPolicy.\n* Unit tests for topology endpoint filtering function, including at least:\n  * No topology constraints (no keys)\n  * A single hard constraint (single specific key)\n  * A single `\"*\"` constraint.\n  * Multiple hard constraints (no final `\"*\"` key).\n  * Multiple constraints plus a final `\"*\"` key.\n* Each of the above sets of keys must have tests with varying topologies\n  available.\n* E2E tests with hard constraint and soft constraints using Endpoints.\n* E2E tests with hard constraint and soft constraints using EndpointSlice.\n\n### Graduation Criteria\n\n#### Alpha\n\n- topologyKeys field is added to the Service API\n- kube-proxy implements service topology in-memory only enabled by `ServiceTopology` feature gate.\n- Unit tests\n\n#### Alpha -\u003e Beta\n\n- scalability concerns are addressed by introducing the `PodLocator` API.\n- kube-proxy is updated to watch `PodLocator` for topology filtering of endpoints.\n- coredns watches `PodLocator` objets and performs topology-aware headless services.\n- E2E tests exist for service topology.\n\n##### Beta -\u003e GA Graduation\n\nTODO: after beta\n"
+  },
+  {
+    "id": "956707cf7b97aeecd1c7f367b6a3d3dc",
+    "title": "Configurable Pod DNS",
+    "authors": ["@MrHohn"],
+    "owningSig": "sig-network",
+    "participatingSigs": ["sig-network"],
+    "reviewers": ["@thockin", "@bowei"],
+    "approvers": ["@thockin", "@bowei"],
+    "editor": "TBD",
+    "creationDate": "2019-01-18",
+    "lastUpdated": "2019-02-11",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Configurable Pod DNS\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Pod API examples](#pod-api-examples)\n    - [Host \u003ccode\u003e/etc/resolv.conf\u003c/code\u003e](#host-)\n    - [Override DNS server and search paths](#override-dns-server-and-search-paths)\n    - [Overriding \u003ccode\u003endots\u003c/code\u003e](#overriding-)\n  - [API changes](#api-changes)\n    - [Semantics](#semantics)\n    - [Invalid configurations](#invalid-configurations)\n  - [Test Plan](#test-plan)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis proposal gives users a way to overlay tweaks into the existing\n`DnsPolicy`. A new PodSpec field `dnsConfig` will contains fields that are\nmerged with the settings currently selected with `DnsPolicy`.\n\n## Motivation\n\nThe `/etc/resolv.conf` in a pod is managed by Kubelet and its contents are\ngenerated based on `pod.dnsPolicy`. For `dnsPolicy: Default`, the `search` and\n`nameserver` fields are taken from the `resolve.conf` on the node where the pod\nis running. If the `dnsPolicy` is `ClusterFirst`, the search contents of the\nresolv.conf is the hosts `resolv.conf` augmented with the following options:\n\n*   Search paths to add aliases for domain names in the same namespace and\n    cluster suffix.\n*   `options ndots` to 5 to ensure the search paths are searched for all\n    potential matches.\n\nThe configuration of both search paths and `ndots` results in query\namplification of five to ten times for non-cluster internal names. This is due\nto the fact that each of the search path expansions must be tried before the\nactual result is found. This order of magnitude increase of query rate imposes a\nlarge load on the kube-dns service. At the same time, there are user\napplications do not need the convenience of the name aliases and do not wish to\npay this performance cost.\n\n### Goals\n\nGiving users a way to overlay tweaks into the existing `DnsPolicy` of a Pod.\n\n### Non-Goals\n\nGiving users a way to configure Pod DNS options on a Cluster level.\n\n## Proposal\n\nThis proposal gives users a way to overlay tweaks into the existing\n`DnsPolicy`. A new PodSpec field `DnsConfig` will contains fields that are\nmerged with the settings currently selected with `DnsPolicy`.\n\nThe fields of `DnsConfig` are:\n\n* `nameservers` is a list of additional nameservers to use for resolution. On\n  `resolv.conf` platforms, these are entries to `nameserver`.\n* `search` is a list of additional search path subdomains. On `resolv.conf`\n  platforms, these are entries to the `search` setting. These domains will be\n  appended to the existing search path.\n* `options` that are an OS-dependent list of (name, value) options. These values\n  are NOT expected to be generally portable across platforms. For containers that\n  use `/etc/resolv.conf` style configuration, these correspond to the parameters\n  passed to the `option` lines. Options will override if their names coincide,\n  i.e, if the `DnsPolicy` sets `ndots:5` and `ndots:1` appears in the `Spec`,\n  then the final value will be `ndots:1`.\n\nFor users that want to completely customize their resolution configuration, we\nadd a new `DnsPolicy: Custom` that does not define any settings. This is\nessentially an empty `resolv.conf` with no fields defined.\n\n### Pod API examples\n\n#### Host `/etc/resolv.conf`\n\nAssume in the examples below that the host has the following `/etc/resolv.conf`:\n\n```bash\nnameserver 10.1.1.10\nsearch foo.com\noptions ndots:1\n```\n\n#### Override DNS server and search paths\n\nIn the example below, the user wishes to use their own DNS resolver and add the\npod namespace and a custom expansion to the search path, as they do not use the\nother name aliases:\n\n```yaml\n# Pod spec\napiVersion: v1\nkind: Pod\nmetadata: {\"namespace\": \"ns1\", \"name\": \"example\"}\nspec:\n  ...\n  dnsPolicy: Custom\n  dnsConfig:\n    nameservers: [\"1.2.3.4\"]\n    searches:\n    - ns1.svc.cluster.local\n    - my.dns.search.suffix\n    options:\n    - name: ndots\n      value: 2\n    - name: edns0\n```\n\nThe pod will get the following `/etc/resolv.conf`:\n\n```bash\nnameserver 1.2.3.4\nsearch ns1.svc.cluster.local my.dns.search.suffix\noptions ndots:2 edns0\n```\n\n#### Overriding `ndots`\n\nOverride `ndots:5` in `ClusterFirst` with `ndots:1`. This keeps all of the\nsettings intact:\n\n```yaml\ndnsPolicy: ClusterFirst\ndnsConfig:\n- options:\n  - name: ndots\n  - value: 1\n```\n\nResulting `resolv.conf`:\n\n```bash\nnameserver 10.0.0.10\nsearch default.svc.cluster.local svc.cluster.local cluster.local foo.com\noptions ndots:1\n```\n\n### API changes\n\n```go\ntype PodSpec struct {\n    ...\n    DNSPolicy string        `json:\"dnsPolicy,omitempty\"`\n    DNSConfig *PodDNSConfig `json:\"dnsConfig,omitempty\"`\n    ...\n}\n\ntype PodDNSConfig struct {\n    Nameservers []string             `json:\"nameservers,omitempty\"`\n    Searches    []string             `json:\"searches,omitempty\"`\n    Options     []PodDNSConfigOption `json:\"options,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"name\"`\n}\n\ntype PodDNSConfigOption struct {\n    Name  string  `json:\"name\"`\n    Value *string `json:\"value,omitempty\"`\n}\n```\n\n#### Semantics\n\nLet the following be the Go representation of the `resolv.conf`:\n\n```go\ntype ResolvConf struct {\n  Nameserver []string // \"nameserver\" entries\n  Searches   []string // \"search\" entries\n  Options    []PodDNSConfigOption  // \"options\" entries\n}\n```\n\nLet `var HostResolvConf ResolvConf` be the host `resolv.conf`.\n\nThen the final Pod `resolv.conf` will be:\n\n```go\nfunc podResolvConf() ResolvConf {\n    var podResolv ResolvConf\n\n    switch (pod.DNSPolicy) {\n    case \"Default\":\n        podResolv = HostResolvConf\n    case \"ClusterFirst\":\n        podResolv.Nameservers = []string{ KubeDNSClusterIP }\n        podResolv.Searches = ... // populate with ns.svc.suffix, svc.suffix, suffix, host entries...\n        podResolv.Options = []PodDNSConfigOption{{\"ndots\",\"5\" }}\n    case \"Custom\": // start with empty `resolv.conf`\n        break\n    }\n\n    // Append the additional nameservers.\n    podResolv.Nameservers = append(Nameservers, pod.DNSConfig.Nameservers...)\n    // Append the additional search paths.\n    podResolv.Searches = append(Searches, pod.DNSConfig.Searches...)\n    // Merge the DnsConfig.Options with the options derived from the given DNSPolicy.\n    podResolv.Options = mergeOptions(pod.Options, pod.DNSConfig.Options)\n\n    return podResolv\n}\n```\n\n#### Invalid configurations\n\nThe follow configurations will result in an invalid Pod spec:\n\n* Nameservers or search paths exceed system limits. (Three nameservers, six\n  search paths, 256 characters for `glibc`).\n* Invalid option appears for the given platform.\n\n### Test Plan\n\nThe following end-to-end test is implemented in addition to unit tests:\n- Create a pod with dns config setup, including nameserver, search path and option.\n- Check if the `resolv.conf` file within the pod is configured properly per the given setup.\n- Send DNS request from the pod and make sure the customized nameserver and\nsearch path are taking effect.\n\n## Graduation Criteria\n\n* Enable by default and soak for 1+ releases\n* Compatible with major systems (e.g. linux, windows)\n\n## Implementation History\n\n* 2017-11-18 - Design proposal merged\n* 2017-12-13 - Alpha with k8s 1.9\n* 2018-03-26 - Beta with k8s 1.10\n* 2019-01-18 - Convert design proposal to KEP\n"
+  },
+  {
+    "id": "c288112f585d6470b5d3d9f16727c7e1",
+    "title": "Graduate Ingress to GA",
+    "authors": ["@bowei"],
+    "owningSig": "sig-network",
+    "participatingSigs": null,
+    "reviewers": ["@aledbf"],
+    "approvers": ["@thockin", "@caseydavenport"],
+    "editor": "",
+    "creationDate": "2018-01-25",
+    "lastUpdated": "2018-04-25",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Graduate Ingress to GA\n\n## Table of contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n- [Design](#design)\n  - [Summary of the proposed changes](#summary-of-the-proposed-changes)\n    - [Potential features for post V1](#potential-features-for-post-v1)\n  - [Path as a prefix](#path-as-a-prefix)\n    - [Paths proposal](#paths-proposal)\n      - [Defaults](#defaults)\n      - [Path matching semantics](#path-matching-semantics)\n      - [\u003ccode\u003eExact\u003c/code\u003e match](#-match)\n      - [\u003ccode\u003ePrefix\u003c/code\u003e match](#-match-1)\n      - [\u003ccode\u003eImplementationSpecific\u003c/code\u003e match](#-match-2)\n      - [Examples](#examples)\n  - [\u003ccode\u003ebackend\u003c/code\u003e to \u003ccode\u003edefaultBackend\u003c/code\u003e](#-to-)\n  - [Hostname wildcards](#hostname-wildcards)\n    - [Hostname proposal](#hostname-proposal)\n      - [Hostname match examples](#hostname-match-examples)\n  - [Status](#status)\n  - [Ingress class](#ingress-class)\n    - [Ingress class proposal](#ingress-class-proposal)\n      - [Interoperability with previous annotation](#interoperability-with-previous-annotation)\n  - [Alternative backend types](#alternative-backend-types)\n    - [Backend types proposal](#backend-types-proposal)\n      - [Backend types examples](#backend-types-examples)\n      - [Supporting custom backends (non-normative)](#supporting-custom-backends-non-normative)\n- [Proposed roadmap](#proposed-roadmap)\n  - [1.14](#114)\n    - [Test plan](#test-plan)\n  - [1.15](#115)\n  - [1.16](#116)\n  - [1.17](#117)\n  - [1.18](#118)\n- [Graduation Criteria](#graduation-criteria)\n  - [API group move to \u003ccode\u003enetworking.k8s.io/v1beta1\u003c/code\u003e](#api-group-move-to-)\n  - [GA](#ga)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n- [Appendix](#appendix)\n  - [Design discussions](#design-discussions)\n  - [Non-options](#non-options)\n  - [Future design: Healthchecks](#future-design-healthchecks)\n    - [Healthchecks proposal](#healthchecks-proposal)\n  - [Potential pre-GA work](#potential-pre-ga-work)\n  - [Rejected designs](#rejected-designs)\n    - [Portable regex for Path](#portable-regex-for-path)\n\u003c!-- /toc --\u003e\n\n## Summary\n\n- Move the Ingress resource from the current API group\n  (extensions.v1beta1) to networking.v1beta1.\n- Graduate the Ingress API with bug fixes to GA.\n\n## Motivation\n\nThe `extensions` API group is considered deprecated.  Ingress is the\nlast non-deprecated API in that group.  All other types have been\nmigrated to other permanent API groups.  Such an API group migration\ntakes three minor version cycles (~9 months) to ensure\ncompatibility. This means any API group movement should be started\nsooner rather than later.\n\nThe Ingress resource has been in a beta state for a *long* time (first\ncommit was in Fall 2015). While the interface [is not\nperfect][survey], there are many [independent implementations][ingress-docs]\nin active use.\n\nWe have a couple of choices (and non-choices, see appendix) for the\ncurrent resource:\n\n1. We can delete the current resource from extensions.v1beta1 in\n  anticipation that an improved API can replace it.\n\n1. We can copy the API as-is (or with minor changes) into\n  networking.v1beta1, preserving/converting existing data (following\n  the same approach taken with all other extensions.v1beta1\n  resources). This will allow us to start the cleanup of the\n  extensions API group. This also prepares the API for GA.\n\nOption 1 does not seem realistic in a short-term time frame (a new API\nwill need to be taken through design, alpha/beta/ga phases). At the\nsame time, there are enough users that the existing API cannot be\ndeleted out right.\n\nIn terms of moving the API towards GA, the API itself has been\navailable in beta for so long that it has attained defacto GA status\nthrough usage and adoption (both by users and by load balancer /\ningress controller providers). Abandoning it without a full\nreplacement is not a viable approach.  It is clearly a useful API and\ncaptures a non-trivial set of use cases.  At this point, it seems more\nprudent to declare the current API as something the community will\nsupport as a V1, codifying its status, while working on either a V2\nIngress API or an entirely different API with a superset of features.\n\n### Goals\n\nA detailed list of the changes being proposed is given in the Design\nsection below.\n\n- Move Ingress to a permanent API group. (status: implemented)\n- Make changes to the Ingress API be in a GA-ready state. (status:\n  proposal).\n  - Clean up the Ingress API (fix ambiguities, API spec bugs).\n  - Promote commonly supported annotations to proper API fields.\n  - Create a suite of conformance tests to validate existing\n    implementations.\n- Make Ingress GA. (status: proposal).\n\n## Design\n\nThis section describes the API fixes proposed for GA.\n\n### Summary of the proposed changes\n\n1. Add path as a prefix and make regex support optional. The current spec\n  states that the path is a regular expression, but support for the flavor\n  defined in the spec varies across providers. In addition, regex matching\n  is not supported by many popular provider implementations.\n1. Fix API field naming:\n   1. `spec.backend` should be called `spec.defaultBackend`.\n1. Hostname wildcard matching. We currently allow for creation of\n   `*.foo.com` and this seems to be a commonly supported host match,\n   but this is not part of the spec.\n1. Formalize the Ingress class annotation into a field and an associated\n   `IngressClass` resource.\n1. Add support for non-Service Backend types.\n\n#### Potential features for post V1\n\nThese are features that were discussed but not part of this discussion:\n\n1. (**POST GA**) Specify healthcheck behavior and be able to configure\n   the healthcheck path and timeout.\n1. (**POST GA**) Improve the Ingress status field to be able to\n   include additional information. The current status currently only\n   contains the provisioned IP address(es) of the load balancer.\n\n### Path as a prefix\n\nThe [current APIs][ingress-api] state that the path is a regular expression\nusing the [POSIX IEEE Std 1003.1 standard][posix-regex]. However, this is not\nconsistent with the syntax supported by any of the common proxy vendors:\n\n| Platform | Syntax                   |\n| nginx    | [PCRE][nginx-re]         |\n| haproxy  | [PCRE/PCRE2][haproxy-re] |\n| envoy    | [ECMAscript][envoy-re]   |\n| skipper  | re2                      |\n\nAmong cloud providers, there is also inconsistent levels of support\nfor regular expression-based path matching. See the load-balancer\ndocumentation for [AWS][aws-re], [GCP][google-re], [Azure][azure-re],\n[Skipper][skipper-link].\n\n[skipper-link]: https://github.com/zalando/skipper\n\nIt is also the case that our [documentation][ingress-docs] (and most\nIngress providers) treats the path match as a prefix match. For\nexample, a narrow interpretation of the specification would require\nall paths to end with `\".*$\"`.\n\nA detailed discussion of this issue can be found\n[here](https://github.com/kubernetes/ingress-nginx/issues/555).\n\n#### Paths proposal\n\n1. Explicitly state the match mode of the path.\n1. Support the existing implementation-specific behavior.\n1. Support a portable prefix match and future expansion of behavior.\n\nAdd a field `ingress.spec.rules.http.paths.pathType` to indicate\nthe desired interpretation of the meaning of the `path`:\n\n```golang\n type HTTPIngressPath struct {\n   ...\n  // Path to match against. The interpretation of Path depends on\n  // the value of PathType.\n  //\n  // Defaults to \"/\" if empty.\n  //\n  // +Optional\n  Path string\n\n  // PathType determines the interpretation of the Path\n  // matching. PathType can be one of the following values:\n  //\n  // Exact  - matches the URL path exactly.\n  //\n  // Prefix - matches based on a URL path prefix split\n  // by '/'. [insert description of semantics described below]\n  //\n  // ImplementationSpecific - interpretation of the Path\n  // matching is up to the IngressClass. Implementations\n  // are not required to support ImplementationSpecific matching.\n  //\n  // +Optional\n  PathType string\n  ...\n }\n ```\n\nV1 validation\n\nNote: default value are permitted between API versions\n([reference][api-conv-versions]).\n\n[api-conv-versions]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#defaulting\n\n##### Defaults\n\nThe `PathType` field will default to a value of `ImplementationSpecific` to\nprovide backwards compatibility.\n\n##### Path matching semantics\n\nFor `Prefix` and `Exact` paths:\n\n1. Let `[p_1, p_2, ..., p_n]` be the list of Paths for a specific host.\n1. Every Path `p_i` must be syntactically valid:\n    1. Must begin with the `'/'` character (relative paths are not allowed by [RFC-7230][rfc7230]).\n    1. Must not contain consecutive `'/'` characters (e.g. `/foo///`, `//`).\n1. For prefix paths, a trailing `'/'` character in the Path is ignored, e.g.\n   `/abc` and `/abc/` specify the same match.\n1. If there is more than one potential match:\n   1. `Exact` match is preferred to a `Prefix` match.\n   1. For multiple prefix matches, the longest Path `p_i` will be the\n      matching path.\n   1. If an `ImplementationSpecific` match exists in the spec, then the\n      preference depends on the implementation.\n1. If there is no matching path, then the `defaultBackend` for the host will be\n   used.\n1. If there is not a match for the host, then the overall `defaultBackend` for\n   the Ingress will be selected.\n\n##### `Exact` match\n\nPath must be exactly the same as the request path.\n\n##### `Prefix` match\n\nMatching is done on a path element by element basis. A path element refers is\nthe list of labels in the path split by the `'/'` separator. A request is a\nmatch for path `p` if every `p` is an element-wise prefix of `p` of the request\npath. Note that if the last element of the path is a substring of the last\nelement in request path, it is *not* a match (e.g. `/foo/bar` matches\n`/foo/bar/baz`, but does not match `/foo/barbaz`).\n\n##### `ImplementationSpecific` match\n\nInterpretation of the implementation-specific behavior is defined by the\nassociated `IngressClass`. Implementations are not required to support this type\nof match. If the match type is not supported, then the controller MAY raise this\nerror as an asynchronous Event to the user.\n\n##### Examples\n\n| Kind   | Path(s)                         | Request path(s)               | Matches?                           |\n| Prefix | `/`                             | (all paths)                   | Yes                                |\n| Exact  | `/foo`                          | `/foo`                        | Yes                                |\n| Exact  | `/foo`                          | `/bar`                        | No                                 |\n| Exact  | `/foo`                          | `/foo/`                       | No                                 |\n| Exact  | `/foo/`                         | `/foo`                        | No                                 |\n| Prefix | `/foo`                          | `/foo`, `/foo/`               | Yes                                |\n| Prefix | `/foo/`                         | `/foo`, `/foo/`               | Yes                                |\n| Prefix | `/aaa/bb`                       | `/aaa/bbb`                    | No                                 |\n| Prefix | `/aaa/bbb`                      | `/aaa/bbb`                    | Yes                                |\n| Prefix | `/aaa/bbb/`                     | `/aaa/bbb`                    | Yes, ignores trailing slash        |\n| Prefix | `/aaa/bbb`                      | `/aaa/bbb/`                   | Yes,  matches trailing slash       |\n| Prefix | `/aaa/bbb`                      | `/aaa/bbb/ccc`                | Yes, matches subpath               |\n| Prefix | `/aaa/bbb`                      | `/aaa/bbbxyz`                 | No, does not match string prefix   |\n| Prefix | `/`, `/aaa`                     | `/aaa/ccc`                    | Yes, matches `/aaa` prefix         |\n| Prefix | `/`, `/aaa`, `/aaa/bbb`         | `/aaa/bbb`                    | Yes, matches `/aaa/bbb` prefix     |\n| Prefix | `/`, `/aaa`, `/aaa/bbb`         | `/ccc`                        | Yes, matches `/` prefix            |\n| Prefix | `/aaa`                          | `/ccc`                        | No, uses default backend           |\n| Mixed  | `/foo` (Prefix), `/foo` (Exact) | `/foo`                        | Yes, prefers Exact                 |\n\n[rfc7230]: https://tools.ietf.org/html/rfc7230#section-5.3.1\n\n### `backend` to `defaultBackend`\n\nThese are straightforward one-to-one renames for better semantic\nmeaning.\n\n| v1beta1 field  | v1                    | rationale                   |\n| `spec.backend` | `spec.defaultBackend` | Explicitly mentions default |\n\nAdd comment clarifying behavior:\n\n\u003e It is up to the controller to resolve conflicts between the defaultBackend's\n\u003e for multiple Ingress definitions that are served from the same\n\u003e VIP if this is possible.\n\n### Hostname wildcards\n\nMost platforms support wildcards for host names, e.g. syntax such as\n`*.foo.com` matches names `app1.foo.com`, `app2.foo.com`. The current\nspec states that `spec.rules.host` must be an exact FQDN match of a\nnetwork host.\n\n#### Hostname proposal\n\nAdd support for a single wildcard `*` as the first label in the hostname.\n\nThe `IngressRule.Host` specification would be changed to:\n\n\u003e `Host` can be \"precise\" which is an domain name without the\n\u003e terminating dot of a network host (e.g. \"foo.bar.com\") or\n\u003e \"wildcard\", which is a domain name prefixed with a single wildcard\n\u003e label (e.g. `\"*.foo.com\"`).\n\u003e\n\u003e Requests will be matched against the `Host` field in the following\n\u003e way:\n\u003e\n\u003e If `Host` is precise, the request matches this rule if the http host\n\u003e header is equal to `Host`.\n\u003e\n\u003e If `Host` is a wildcard, then the request matches this rule if the\n\u003e http host header is to equal to the suffix (removing the first\n\u003e label) of the wildcard rule.\n\u003e\n\u003e - The wildcard character `'*'` must appear by itself as the first\n\u003e   DNS label and matches only a single label.\n\u003e - You cannot have a wildcard label by itself (e.g. `Host == \"*\"`).\n\n##### Hostname match examples\n\n- `\"*.foo.com\"` matches `\"bar.foo.com\"` because they share an the same\n   suffix `\"foo.com\"`.\n- `\"*.foo.com\"` does not match `\"aaa.bbb.foo.com\"` as the wildcard only\n   matches a single label.\n- `\"*.foo.com\"` does not match `\"foo.com\"`, as the wildcard must match a\n   single label.\n\nNote: label refers to a \"DNS label\", i.e. the strings separated by the dots \".\"\nin the domain name.\n\n### Status\n\nAs this is strictly additive, this could be punted to post-GA to reduce\nthe size of the change.\n\n### Ingress class\n\nThe `kubernetes.io/ingress.class` annotation is required for selecting between\nmultiple Ingress providers. As support for this annotation is universal, this\nconcept should be promoted to an actual field.\n\n#### Ingress class proposal\n\nPromoting the annotation as it is currently defined as an opaque string is the\nmost direct path but precludes any future enhancements to the concept.\n\nAn alternative is to create a new resource `IngressClass` to take its place.\nThis resource will serve a couple of purposes:\n\n- Define the set of valid classes available to the user. Gives operators control\n  over allowed classes.\n- Allow us to evolve the API to express concepts such a levels of service\n  associated with a given Ingress controller.\n\nAdd a field to `ingress.spec`:\n\n```golang\ntype IngressSpec struct {\n  ...\n  // Class is the name of the IngressClass cluster resource. This defines\n  // which controller(s) will implement the resource.\n  Class string\n  ...\n}\n\n...\n\n// IngressClass represents the class of the Ingress, referenced by the\n// ingress.spec. IngressClass will be a non-namespaced Cluster resource.\ntype IngressClass struct {\n  metav1.TypeMeta\n  metav1.ObjectMeta\n\n  // Controller is responsible for handling this class. This should be\n  // specified as a domain-prefixed path, e.g. \"acme.io/ingress-controller\".\n  //\n  // This allows for different \"flavors\" that are controlled by the same\n  // controller. For example, you may have different Parameters for\n  // the same implementing controller.\n  Controller string\n\n  // Parameters is a link to a custom resource configuration for\n  // the controller. This is optional if the controller does not\n  // require extra parameters.\n  //\n  // +optional\n  Parameters *TypeLocalObjectReference\n}\n```\n\n##### Interoperability with previous annotation\n\nThe Ingress class set in the annotation takes priority to the\n`Spec.Class` field. The controller MAY emit a warning event if the\nuser sets conflicting (different) values for the annotation and\n`Spec.Class`.\n\n### Alternative backend types\n\nThe Ingress resource is an L7 description of a composite set of\nservices. It currently supports only Kubernetes Services as a\nbackends. However, there are many use cases where a portion of the\nHTTP requests could be routed to a different kind of resource. For\nexample, serving content from an object storage ([S3][s3-backend],\n[GCS][gcs-backend]) is a commonly requested feature.\n\nAt the same time, we do not expect to enumerate all possible backends\nthat could arise, nor do we expect that naming of the resources will\nbe uniform in schema, parameters etc. Similarly, many of the resources\nwill be implementation-specific.\n\n#### Backend types proposal\n\nAdd a field to the `IngressBackend` struct with an object reference:\n\n```golang\ntype IngressBackend struct {\n  // Only one of the following fields may be specified.\n\n  // Service references a Service as a Backend. This is specially\n  // called out as it is required to be supported AND to reduce\n  // verbosity.\n  // +optional\n  Service *ServiceBackend\n\n  // Resource is an ObjectRef to another Kubernetes resource in the namespace\n  // of the Ingress object.\n  // +optional\n  Resource *v1.TypedLocalObjectReference\n}\n\n// ServiceBackend references a Kubernetes Service as a Backend.\ntype ServiceBackend struct {\n  // Service is the name of the referenced service. The service must exist in\n  // the same namespace as the Ingress object.\n  // +optional\n  Name string\n\n  // Port of the referenced service. If unspecified and the ServiceName is\n  // non-empty, the Service must expose a single port.\n  // +optional\n  Port ServiceBackendPort\n}\n\n// ServiceBackendPort is the service port being referenced.\ntype ServiceBackendPort struct {\n  // Number is the numerical port number (e.g. 80) on the Service.\n  Number int\n  // Name is the name of the port on the Service.\n  Name string\n}\n```\n\nSupport for non-`Service` type `Resource`s is\nimplementation-specific. Implmentations MUST support Kubernetes\nService. Support for other types is OPTIONAL.\n\n##### Backend types examples\n\nIngress routing everything to `foo-app`:\n\n```yaml\nkind: Ingress\nspec:\n  class: acme-lb\n  backend:\n    service:\n\t  name: foo-app\n\t  port:\n\t    number: 80\n```\n\nIngress routing everything to the ACME storage bucket:\n\n```yaml\nkind: Ingress\nspec:\n  class: acme-lb\n  backend:\n    resource:\n\t  apiGroup: acme.io/networking\n\t  kind: storage-bucket\n\t  name: foo-bucket\n```\n\nInvalid configuration (uses both resource and service):\n\n```yaml\nkind: Ingress\nspec:\n  class: acme-lb\n  backend:\n    service:\n\t  name: foo-app\n\t  port:\n\t    number: 80\n    resource: # INVALID!\n\t  apiGroup: acme.io/networking\n\t  kind: storage-bucket\n\t  name: foo-bucket\n```\n\n##### Supporting custom backends (non-normative)\n\nAs a sketch, an object bucket can be named with a CRD. NOTE: this\nexample is non-normative and for illustration purposes only.\n\n```golang\ntype Bucket struct {\n  metav1.TypeMeta\n  metav1.ObjectMeta\n  Spec BucketSpec\n}\n\ntype BucketSpec struct {\n  Bucket string\n  Path   string\n}\n```\n\nThe associated `IngressBackend` referencing the bucket would be:\n\n```yaml\nbackend:\n  resource:\n    apiGroup: bucket.io\n    kind: bucket\n    name: my-bucket\n```\n\n## Proposed roadmap\n\n### 1.14\n\n- [x] Copy the Ingress API to `networking.k8s.io/v1beta1` (preserving\n  existing data and round-tripping with the extensions Ingress API,\n  following the approach taken for all other `extensions/v1beta1`\n  resources).\n- [x] Develop a set of planned changes and GA graduation criteria with\n  sig-network (intent is to target a minimal set of bugfixes and\n  non-breaking changes)\n- [x] Announce `extensions/v1beta1` Ingress as deprecated (and\n      announce plan for GA)\n\n#### Test plan\n\n- Copy existing Ingress tests, changing the resource type to the new\n  group. Keep existing tests as is.\n\n### 1.15\n\n- [x] Update API server to persist in networking.k8s.io/v1beta1 kubernetes/kubernetes#77139\n- [x] Update in-tree controllers, examples, and clients to target kubernetes/kubernetes#77617\n  `networking.k8s.io/v1beta1`\n- [x] Update Ingress controllers in the kubernetes org to target\n  `networking.k8s.io/v1beta1`\n  - [x] [ingress-nginx](https://github.com/kubernetes/ingress-nginx/pull/4127)\n  - [x] [ingress-gce](https://github.com/kubernetes/ingress-gce/issues/770)\n- [x] Update documentation to recommend new users start with kubernetes/website#14239\n  networking.k8s.io/v1beta1, but existing users stick with\n  `extensions/v1beta1` until `networking.k8s.io/v1` is available.\n- [x] Update documentation to reference `networking.k8s.io/v1beta1` kubernetes/website#14239\n\n### 1.16\n\n- [ ] Meet graduation criteria and promote API to `networking.k8s.io/v1`\n- [ ] Implement API changes to GA version.\n- [ ] Announce `networking.k8s.io/v1beta1` Ingress as deprecated\n\n### 1.17\n\n- [ ] Update API server to persist in `networking.k8s.io/v1`.\n- [ ] Update in-tree controllers, examples, and clients to target\n  `networking.k8s.io/v1`.\n- [ ] Update Ingress controllers in the kubernetes org to target\n  `networking.k8s.io/v1`.\n- [ ] Update documentation to reference `networking.k8s.io/v1`.\n- [ ] Evangelize availability of v1 Ingress API to out-of-org Ingress\n      controllers\n\n### 1.18\n\n- [ ] Remove ability to serve `extensions/v1beta1` and\n  `networking.k8s.io/v1beta1` Ingress resources (preserve ability to\n  read existing `extensions/v1beta1` Ingress objects from storage and\n  serve them via the `networking.k8s.io/v1` API)\n\n## Graduation Criteria\n\n### API group move to `networking.k8s.io/v1beta1`\n\n- [x] 1.14: Ingress API exists and has parity with existing\n  `extensions/v1beta1` API\n- [x] 1.14: `extensions/v1beta1` Ingress tests are replicated against\n  `networking.k8s.io`\n- [x] 1.15: all in-tree use and in-org controllers switch to\n  `networking.k8s.io` API group\n- [ ] 1.15: documentation and examples are updated to refer to\n  networking.k8s.io API group `networking.k8s.io/v1`\n\n### GA\n\n- [ ] 1.17: API finalized and implemented on the branch.\n- [ ] 1.XX: Ingress spec and conformance tests finalized and running against branch.\n- [ ] 1.XX: API changes merged into the main API, with tests from v1beta1 pointing to GA.\n\n## Implementation History\n\n- 1.14: Copied Ingress API to the networking API group.\n\n## Alternatives\n\nSee motivation section.\n\n## Appendix\n\n### Design discussions\n\n- Kubecon EU 2019 [sig-network meetup][kubecon-eu-2019].\n\n[kubecon-eu-2019]: https://docs.google.com/document/d/1x8KoNWLKA9JEDD-z88A8Kb1yzHJ8YDhDCoBZsQxMM6Q/edit#bookmark=id.60xvqkshg3z4\n\n### Non-options\n\nOne suggestion was to move the API into a new API group, defined as a\nCRD.  This does not work because there is no way to do round-trip of\nexisting Ingress objects to a CRD-based API.\n\n### Future design: Healthchecks\n\nThe current spec does not have any provisions to customize\nhealthchecks for referenced backends. Many users already have a\nhealthcheck URL that is lightweight and different from the HTTP root\n(i.e. `/`).\n\nOne obvious question that arises is why the Ingress healthcheck\nconfiguration is (a) is needed and (b) is different from the current\nPod readiness and liveness checks. The Ingress healthcheck represents\nan end-to-end check from the proxy server to the backend.  The\nKubelet-based service health check operates only within the VM and\ndoes not include the network path. A minor point is that it is also\nthe case that some providers require a healthcheck to be specified as\npart of load balancing.\n\nAn option that has been explored is to infer the healthcheck URL from\nthe Readiness/Liveness probes on the Pods of the Service. This method\nhas proven to be unworkable: Every Pod in a Service can have a\ndifferent Readiness probe definition and therefore it's not clear\nwhich one should be used. Furthermore, the behavior is implicit and\ncreates action-at-a-distance relationship between the Ingress and Pod\nresources.\n\n#### Healthchecks proposal\n\nAdd the following fields to `IngressBackend`:\n\n```golang\ntype IngressBackend struct {\n  ...\n  // Healthcheck defines custom healthcheck for this backend.\n  // +optional\n  Healthcheck *IngressBackendHealthcheck\n}\n\ntype IngressBackendHealthcheck struct {\n  // HTTP defines healthchecks using the HTTP protocol.\n  HTTP *IngressBackendHTTPHealthcheck\n}\n\n// IngressBackendHTTPHealthcheck is a healthcheck using the HTTP protocol.\ntype IngressBackendHTTPHealthcheck struct {\n  // Host header to send when healthchecking. If empty, the host header will be\n  // implementation specific.\n  Host string\n  // Path to use for the HTTP healthcheck. If empty, the root '/' path will be\n  // used for healthchecking.\n  Path string\n  // TimeoutSeconds for the healthcheck. Failure to respond with a success code\n  // within TimeoutSeconds will be counted towards the FailureThreshold.\n  TimeoutSeconds int\n  // FailureThreshold is the number of consecutive failures necesseary to\n  // indicate a backend failure.\n  FailureThreshold int\n}\n```\n\nIf `Healthcheck` is nil, then the implementation default healthcheck will be\nconfigured, healthchecking the root `/` path. If `Healthcheck` is specfied,\nthen the backend health will be checked using the parameters listed above.\n\n### Potential pre-GA work\n\nNote: these items are NOT the main focus of this KEP, but recorded\nhere for reference purposes. These items came up in discussions on the\nKEP (roughly sorted by practicality):\n\n- Spec path as a prefix, maybe as a new field\n- Rename `backend` to `defaultBackend` or something more obvious\n- Be more explicit about wildcard hostname support (I can create *.bar.com but\n  in theory this is not supported)\n- Add health-checks API\n- Specify whether to accept just HTTPS or also allow bare HTTP\n- Better status\n- Formalize Ingress class\n- Reference a secret in a different namespace?  Use case: avoid copying wildcard\n  certificates (generated with cert-manager for instance)\n- Add non-required features (levels of support)\n- Some way to have backends be things other than a service (e.g. a GCS bucket)\n- Some way to restrict hostnames and/or URLs per namespace\n- HTTP to HTTPS redirects\n- Explicit sharing or non-sharing of external IPs (e.g. GCP HTTP LB)\n- Affinity\n- Per-backend timeouts\n- Backend protocol\n- Cross-namespace backends\n\n### Rejected designs\n\nThis section contains rejected design proposals for future reference.\n\n#### Portable regex for Path\n\nThe safest route for specifying the regex would be to state a limited\nsubset that can be used in a portable way. Any expressions outside of\nthe subset will have implementation specific behavior.\n\nRegular expression subset (derived from [re2][re2-syntax] syntax page)\n\n| Expression | description             |\n| `.`        | any character           |\n| `[xyz]`    | character class         |\n| `[^xyz]`   | negated character class |\n| `x*`       | 0 or more x's           |\n| `x+`       | 1 or more x's           |\n| `xy`       | x followed by y         |\n| `x|y`      | x or y (prefer x)       |\n| `(abc)`    | grouping                |\n\nMaintaining a regular expression subset is not worth the complexity and\nis likely impossible across the [many implementations][regex-survey].\n\n\u003c!-- References --\u003e\n\n[aws-re]: https://docs.aws.amazon.com/elasticloadbalancing/latest/application/listener-update-rules.html\n[azure-re]: https://docs.microsoft.com/en-us/azure/application-gateway/application-gateway-create-url-route-portal\n[envoy-re]: https://github.com/envoyproxy/envoy/blob/v1.10.0/api/envoy/api/v2/route/route.proto#L334\n[gcs-backend]: TODO\n[google-re]: https://cloud.google.com/load-balancing/docs/https/url-map-concepts\n[haproxy-re]: http://git.haproxy.org/?p=haproxy-1.9.git;a=blob;f=Makefile;h=0814440e48d57ae53f058ffb3f233c80b63871f2;hb=HEAD#l17\n[ingress-api]: https://github.com/kubernetes/api/blob/release-1.14/networking/v1beta1/types.go#L170\n[ingress-docs]: https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-controllers\n[nginx-re]: http://nginx.org/en/docs/http/server_names.html#regex_names\n[posix-regex]: https://www.boost.org/doc/libs/1_38_0/libs/regex/doc/html/boost_regex/syntax/basic_extended.html\n[re2-syntax]: https://github.com/google/re2/wiki/Syntax\n[s3-backend]: TODO\n[survey]: https://github.com/bowei/k8s-ingress-survey-2018\n[regex-survey]: https://en.wikipedia.org/wiki/Comparison_of_regular_expression_engines\n"
+  },
+  {
+    "id": "6e7fcd8076af94df6d295f7d162461e5",
+    "title": "Remove kube-proxy's automatic clean up logic",
+    "authors": ["@vllry"],
+    "owningSig": "sig-network",
+    "participatingSigs": null,
+    "reviewers": ["@andrewsykim"],
+    "approvers": ["@bowei", "@thockin"],
+    "editor": "TBD",
+    "creationDate": "2018-03-24",
+    "lastUpdated": "2018-04-02",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Remove kube-proxy's automatic clean up logic\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-goals](#non-goals)\n- [Proposal](#proposal)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nRemove any clean up functionality in kube-proxy that is done automatically/implicitly.\nMore concretely, kube-proxy should only attempt to clean up its proxy rules when the `--cleanup` flag is set.\n\n## Motivation\n\nHistorically, kube-proxy has been built with a \"clean up\" functionality that allows it to find and remove any proxy\nrule that it created. This functionality runs in two steps. The first step requires an explicit request from\nthe user by setting the `--cleanup` flag in which case kube-proxy would attempt to remove proxy rules associated with\nall its proxy modes and then exit. The second step is done automatically by kube-proxy on start-up in which kube-proxy\nwould detect and remove any proxy rules associated with the proxy modes it was not currently using.\n\nWe've learned by now that trying to automatically clean up proxy rules is prone to bugs due to the complex dependency\nbetween all the proxy modes and the Linux kernel. There are overlapping proxy rules between the iptables\nand IPVS proxy modes which can result in connectivity issues during start-up as kube-proxy attempts to clean up\nthese overlapping rules. In the worse case scenario, kube-proxy will flush active iptable chains every time it is restarted.\nThis KEP aims to simplify this clean up logic, and as a result, make kube-proxy's behavior more predictable and safe.\n\n### Goals\n\n* Simplify the clean up code path for kube-proxy by removing any logic that is done implicitly by kube-proxy.\n* Remove bugs due to overlapping proxier rules, causing user-visible connectivity issues during kube-proxy startup.\n\n### Non-goals\n\n* Re-defining the set of proxy rules that should be cleaned up.\n* Improving the performance of kube-proxy startup.\n\n## Proposal\n\nOnly clean up iptables and IPVS proxy rules if `--cleanup` is set.\nIf `--cleanup` is not set, start kube-proxy without cleaning up any existing rules.\n\nFor example, remove code like [this](https://github.com/kubernetes/kubernetes/blob/e7eb742c1907eb4f1c9e5412f6cd1d4e06f3c277/cmd/kube-proxy/app/server_others.go#L180-L187) where kube-proxy attempts to clean up proxy rules in a proxy mode it is not using.\n\n```go\n    if proxyMode == proxyModeIPTables {\n\t    ...\n\t    userspace.CleanupLeftovers(iptInterface)\n\t    // IPVS Proxier will generate some iptables rules, need to clean them before switching to other proxy mode.\n\t    // Besides, ipvs proxier will create some ipvs rules as well.  Because there is no way to tell if a given\n\t    // ipvs rule is created by IPVS proxier or not.  Users should explicitly specify `--clean-ipvs=true` to flush\n\t    // all ipvs rules when kube-proxy start up.  Users do this operation should be with caution.\n\t    if canUseIPVS {\n\t\t    ipvs.CleanupLeftovers(ipvsInterface, iptInterface, ipsetInterface, cleanupIPVS)\n\t    }\n\t    ...\n```\n\nWith proxy clean up always requiring an explicit request from the user, we will also recommend users to reboot their nodes if they choose to\nswitch between proxy modes. This was always expected but not documented/vocalized well enough. The `--cleanup` flag should only be used in the event that a\nnode reboot is not an option. Accompanying a proxy mode switch with a node reboot ensures that any state in the kernel associated with the previous\nproxy mode is cleared. This expectation should be well documented in the release notes, the Kubernetes docs, and in kube-proxy's help command.\n\n## Graduation Criteria\n\n* kube-proxy does not attempt to remove proxy rules in any proxy mode unless the `--cleanup` flag is set.\n* the expectations around kube-proxy's clean up behavior is stated clearly in the release notes, docs and in kube-proxy's help command.\n* there is documentation strongly suggesting that a node should be rebooted along with any proxy mode switch\n\n## Implementation History\n\n* 2019-03-14: initial [bug report](https://github.com/kubernetes/kubernetes/issues/75360) was created where kube-proxy would flush iptable chains even when the proxy mode was iptables\n\n## Alternatives\n\n* Removing support for cleaning up other proxy modes has been suggested. For example, if a user wished to change proxy modes,\nrather than running `kube-proxy --cleanup`, users would be advised to restart the worker node.\n* Continue to support automatic/implicit clean up of proxy rules by fixing any existing bugs where overlapping rules are removed.\nThough we can continue to support this, doing so correctly is not trivial. The overlapping rulesets between the existing\nproxy modes makes it difficult to fix this without adding more complexity to kube-proxy or changing the proxy rules in an incompatible way.\n"
+  },
+  {
+    "id": "db3159c422e166f0f35b816cd2875d35",
+    "title": "Finalizer Protection for Service LoadBalancers",
+    "authors": ["@MrHohn"],
+    "owningSig": "sig-network",
+    "participatingSigs": ["sig-cloud-provider"],
+    "reviewers": ["@andrewsykim", "@bowei", "@jhorwit2", "@jiatongw"],
+    "approvers": ["@andrewsykim", "@bowei", "@thockin"],
+    "editor": "TBD",
+    "creationDate": "2019-04-23",
+    "lastUpdated": "2019-04-29",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Finalizer Protection for Service LoadBalancers\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n- [Proposal](#proposal)\n  - [Risks and Mitigations](#risks-and-mitigations)\n    - [n+2 upgrade/downgrade is not supported](#n2-upgradedowngrade-is-not-supported)\n  - [Other notes](#other-notes)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nWe will be adding finalizer protection to ensure the Service resource is not\nfully deleted until the correlating load balancer resources are deleted. Any\nservice that has `type=LoadBalancer` (both existing and newly created ones)\nwill be attached a service LoadBalancer finalizer, which should be removed by\nservice controller upon the cleanup of related load balancer resources. Such\nfinalizer protection mechanism will be released with phases to ensure downgrades\ncan happen safely.\n\n## Motivation\n\nThere are various cases where service controller can leave orphaned load\nbalancer resources after services are deleted (ref discussion on\nhttps://github.com/kubernetes/kubernetes/issues/32157,\nhttps://github.com/kubernetes/kubernetes/issues/53451). We are periodically\ngetting bug reports and customer issues that replicated such problem, which\nseems to be common enough and is worth to have a better mechanism for ensuring\nthe cleanup of load balancer resources.\n\n### Goals\n\nEnsure the Service resource is not fully deleted until the correlating load\nbalancer resources are deleted.\n\n## Proposal\n\nWe are going to define a finalizer for service LoadBalancers with name\n`service.kubernetes.io/load-balancer-cleanup`. This finalizer will be attached\nto any service that has `type=LoadBalancer` if the cluster has the cloud\nprovider integration enabled. Upon the deletion of such service, the actual\ndeletion of the resource will be blocked until this finalizer is removed.\nThis finalizer will not be removed until cleanup of the correlating load\nbalancer resources are considered finished by service controller.\n\nNote that the removal of this finalizer might also happen when service type\nchanges from `LoadBalancer` to another. This however doesn't change the\nimplication that the resources cleanup must be fulfilled before fully deleting\nthe service.\n\nThe lifecyle of a `LoadBalancer` type service with finalizer would look like:\n- Creation\n  1. User creates a service.\n  2. Service controller observes the creation and attaches finalizer to the service.\n  3. Provision of load balancer resources.\n- Deletion\n  1. User issues a deletion for the service.\n  2. Service resource deletion is blocked due to the finalizer.\n  3. Service controller observed the deletion timestamp is added.\n  4. Cleanup of load balancer resources.\n  5. Service controller removes finalizer from the service.\n  6. Service resource deleted.\n- Update to another type\n  1. User update service from `type=LoadBalancer` to another.\n  2. Service controller observed the update.\n  3. Cleanup of load balancer resources.\n  4. Service controller removes finalizer from the service.\n\nThe expected cluster upgrade/downgrade path for service with finalizer would be:\n- Upgrade from pre-finalizer version\n  - All existing `LoadBalancer` services will be attached a finalzer upon startup\n  of the new version of service controller.\n  - The newly created `LoadBalancer` services will have finalizer attached upon\n  creation.\n- Downgrade from with-finailzer version\n  - All existing `LoadBalancer` service will have the attached finalizer removed\n  upon the cleanup of load balancer resources.\n  - The newly created `LoadBalancer` services will not have finailzer attached.\n\nTo ensures that downgrades can happen safely, the first release will include the\n\"remove finalizer\" logic with the \"add finalizer\" logic behind a gate. Then in a\nlater release we will remove the feature gate and enable both the \"remove\" and\n\"add\" logic by default.\n\nAs such, we are proposing Alpha/Beta/GA phase for this enhancement as below:\n- Alpha: Finalizer cleanup will always be on. Finalizer addition will be off by\ndefault but can be enabled via a feature gate.\n- Beta: Finalizer cleanup will always be on. Finalizer addition will be on by\ndefault but can be disabled via a feature gate.\n- GA: Service LoadBalancers Finalizer Protection will always be on.\n\n### Risks and Mitigations\n\n#### n+2 upgrade/downgrade is not supported\n\nIf user does n+2 upgrade from v1.14 -\u003e v1.16 and then does a downgrade back to v1.14.\nThey would have added finalizers to the Service but then lose the removal logic on\nthe downgrade. And hence Service with `type=LoadBalancer` can't be deleted until the\nfinalizer on it is manually removed.\n\nTo keep the upgrade/downgrade safe a user would always do n+1 upgrade/downgrade as\nstated on https://kubernetes.io/docs/setup/version-skew-policy/#supported-component-upgrade-order.\n\n### Other notes\n\nIf the cloud provider opts-out of [LoadBalancer](https://github.com/kubernetes/cloud-provider/blob/402566916174f020983cb0bd467daeae6206ae02/cloud.go#L48-L49)\nsupport, service controller won't be run at all (see [here](https://github.com/kubernetes/kubernetes/blob/3e52ea8081abc13398de6283c31056cd6aecf6b4/pkg/controller/service/service_controller.go#L229-L232)).\nHence finalizer won't be added/removed by service controller.\n\nIf any other custom controller that watches Service with `type=LoadBalancer`, it\nshould implement its own finalizer protection.\n\n### Test Plan\n\nWe will implement e2e test cases to ensure:\n- Service finalizer protection works with various service lifecycles on a cluster\nthat enables it.\n\nIn addition to above, we should have upgrade/downgrade tests that:\n- Verify the downgrade path and ensure service finalizer removal works.\n- Verify the upgrade path and ensure finalizer protection works with existing LB\nservices. \n\n### Graduation Criteria\n\nBeta: Allow Alpha (\"remove finalizer\") to soak for at least one release, then\nswitch the \"add finalizer\" logic to be on by default.\n\nGA: Allow Beta to soak for at least one release. (There is no behavioral\ndifferences from the Beta phase.)\n\n## Implementation History\n\n- 2017-10-25 - First attempt of adding finalizer to service\n(https://github.com/kubernetes/kubernetes/pull/54569)\n- 2018-07-06 - Split finalizer cleanup logic to a separate PR\n(https://github.com/kubernetes/kubernetes/pull/65912)\n- 2019-04-23 - Creation of the KEP\n- 2019-05-23 - PR merged for adding finalizer support in LoadBalancer services (https://github.com/kubernetes/kubernetes/pull/78262)\n"
+  },
+  {
+    "id": "1a3dfedf24b812a435691fafd952eb4e",
+    "title": "graduate-nodelocaldns-to-beta",
+    "authors": ["@prameshj"],
+    "owningSig": "sig-network",
+    "participatingSigs": null,
+    "reviewers": ["@bowei", "@thockin", "@johnbelamaric"],
+    "approvers": ["@bowei", "@thockin"],
+    "editor": "",
+    "creationDate": "2019-04-24",
+    "lastUpdated": "2019-04-25",
+    "status": "implementable",
+    "seeAlso": ["/keps/sig-network/0030-nodelocal-dns-cache.md"],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Graduate NodeLocal DNSCache to beta\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n- [Proposal](#proposal)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Alternatives](#alternatives)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\n[NodeLocal DNSCache](https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/dns/nodelocaldns/README.md) is an addon that runs a dnsCache pod as a daemonset to improve clusterDNS performance and reliability. The feature has been in Alpha since 1.13 release. This document lays out the plan to promote it to beta.\n\n## Motivation\n\nNodeLocal DNSCache has been in Alpha for the past 2 releases and users have deployed it to fix DNS performance issues.\n- [Example 1](https://github.com/kubernetes/kubernetes/issues/56903#issuecomment-485353223)\n- [Example 2](https://github.com/kubernetes/kubernetes/issues/45363#issuecomment-443019910)\n\nBased on the feedback so far, we feel the feature is ready to be graduated to beta.\n\n### Goals\n\nGraduate NodeLocal DNSCache to beta.\n\n## Proposal\n\nN.B. Although CoreDNS is now the default DNS server on Kubernetes clusters, this document still uses the name kube-dns since the service name is still the same.\n\nBased on the initial feedback for NodeLocal DNSCache feature, HA seems to be the common ask. \n\nThe current implementation introduces a single point of failure, since all pods on a node rely on the node-local-dns pod that is running on the same node, for DNS requests.\nAny dedicated node-agent brings with it the issue of single-point of failure. Example: kube-proxy or any other CNI pod has a similar issue. These are in the control plane and they do leave behind some state upon exit that is sufficient for services to partially work. In that way, the scenario is a little different from the a node-cache pod going down, since the latter in in the data path. But it is not all that different since kube-proxy if down for long enough, will cause a drift from current configured state resulting in datapath failures.\n\nHere are some failure modes for the node-cache pod:\n\n1) Pod Evicted - We create this daemonset with `priorityClassName: system-node-critical` setting to greatly reduce the likelihood of eviction.\n2) Config error - node-local-dns restarts repeatedly due to incorrect config. This will be resolved only when the config error has been fixed. There will be DNS downtime until then, even though the kube-dns pods might be available.\n3) OOMKilled - node-local-dns gets OOMKilled due to its own memory usage or some other component using up all memory resources on the node. There is a chance this will cause other disruptions on the node in addition to DNS downtime though. \n4) Upgrades to node-local-dns daemonset - There will be DNS downtime when node-local-dns pods shut down, until the new pods are up and running.\n\nWe are proposing a solution that will help in all these cases. For beta, we will start providing enablement for HA, full implementation will be a GA criterion.\n \nThe proposal here is to use an additional listen IP for node-local-dns pod. The node-local-dns pod listens on the 169.254.20.10 IP address today. We will extend node-local-dns to listen on the kube-dns service IP as well. Requests to kube-dns service IP will be handled by node-local-dns pod when it is up. If it is unavailable, the requests will go to kube-dns endpoints instead. The determination of whether node-local-dns service is available will be done by an external component - This could be a new daemonset or new functionality in an existing daemonset that manages networking.\n\n### Risks and Mitigations\n\n* The proposed HA solution will not work in IPVS mode of kube-proxy. This is because skipping IPVS translation rules is not possible using an iptables NOTRACK rule. So, if the service IP is used by pods for DNS resolution, requests will always hit the IPVS load-balancing rules and reach the kube-dns endpoints.\nOne way to get the desired behavior is to change the selectors for the kube-dns service, so that we set empty endpoints when we want the node-local-dns pods to be used. However, this approach has not been tested yet. Also, this would change it across the board, instead of for a single node as the need may be.\n\n* If the pod performing the checks and flipping the DNS server gets evicted, we could still end up with DNS downtime.\n\n* Adding a watcher introduces more resource consumption to support NodeLocal DNSCache feature. This can be mitigated by combining this logic into an existing daemonset. Also, it will be possible to run node-local-dns without this additional component, without the HA benefit.\n\n## Design Details\n\nIn this new design, node-local-dns pod creates a dummy interface with 2 IP addresses - the link local IP address 169.254.20.10(This happens already today) and the kube-dns service IP address. \nA new service spec is added to the node-local-dns yaml, this is almost identical to the kube-dns service spec. \n\n```\napiVersion: v1\nkind: Service\nmetadata:\n  name: node-local-upstream\n  namespace: kube-system\n  labels:\n    k8s-app: kube-dns\n    kubernetes.io/cluster-service: \"true\"\n    addonmanager.kubernetes.io/mode: Reconcile\n    kubernetes.io/name: \"NodeLocalUpstream\"\nspec:\n  selector:\n    k8s-app: kube-dns\n  ports:\n  - name: dns\n    port: 53\n    protocol: UDP\n  - name: dns-tcp\n    port: 53\n```\n\nIt has the same selectors as kube-dns, so this service IP will be mapped to the same kube-dns endpoints.\nThis new service is required for node-local-dns pod to talk to kube-dns endpoints. This will be the IP address used by node-local-dns in case of cache misses in the cluster.local domain.\nThis service spec does not reserve a specific clusterIP, let's assume the assigned IP is 10.0.0.50 (it can be different on each setup). Let's assume the kube-dns IP is 10.0.0.10.\nBy default, kube-proxy will install rules so that packets targeting 10.0.0.10 are DNAT'ed to one of the kube-dns endpoints. A similar rule will be installed for 10.0.0.50 as well. However, we need packets to 10.0.0.10 to be sent to the local interface that node-local-dns pod is listening on. This is possible by using the NOTRACK action in iptables.\n\nAs mentioned in the previous [KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/0030-nodelocal-dns-cache.md#proposal), node-local-dns pod installs iptables rule with NOTRACK action so that connections to and from the node-local-dns IP on port 53 can avoid being tracked via CONNTRACK. The purpose was to prevent usage of conntrack entries for DNS requests. Another benefit is that this avoids additional NAT table rules from being applied on the packet. So, as long as we have a rule \n`-d 10.0.0.10 --dport 53 -j NOTRACK`,\nthe NAT table rules that reroute the packet to kube-dns endpoints will not be applied. We also need a filter table rule to make sure the packet isn't dropped.\nThe request packet can now be locally consumed by node-local-dns pod. The node-local-dns pod will use the new service IP - 10.0.0.50, as its Upstream Nameserver, which will still map to the endpoints.\nSo, we can use the NOTRACK rule as a switch to flip between using node-local-dns and kube-dns endpoints.\n\nHere is a diagram to explain this flow:\n\n\n![ ](nodelocal-HA.png  \"NodeLocalDNS HA-design\")\n\n\nAs summarized above, we will use the iptables NOTRACK rule to implement the \"USE LOCAL?\" condition in the diagram.\n\nThe benefits of this approach are:\n\n1) node-local-dns can be used on existing clusters without any kubelet change. Pods continue to use kube-dns service IP in their /etc/resolv.conf and we transparently switch the backend to the new cache.\n\n2) We are able to, somewhat elegantly, failover to kube-dns endpoints.\n\n3) Disabling node-local-dns does not require any kubelet change either.\n\nWe still need some component to dynamically determine when to use node-local-dns and when to flip to kube-dns endpoints. This logic can be separated out into an independent container/pod whose function is to query for dns records on 169.254.20.10:53 and follow some threshold to either install or remove the NOTRACK rules. This can be a new Daemonset or combined into an existing Daemonset that is in HostNetwork mode and manages iptables rules in some way - for instance a CNI Daemonset. This component will handle adding all iptables rules needed for node-local-dns.\n\nThe caveat of this approach is that it only works in the iptables implementation of kube-proxy. \nAnother observation is that the upstream dns server IP used by node-local-dns will differ from one setup to another since it is a dynamically allocated service IP.  This doesn't appear to be a major concern.\n\n### Test Plan\n\n* We are running all the existing DNS tests with NodeLocal DNSCache enabled:\n  - [kube-dns-performance-nodecache](https://k8s-testgrid.appspot.com/sig-network-gce#gce-kubedns-performance-nodecache)\n  - [coredns-performance-nodecache](https://k8s-testgrid.appspot.com/sig-network-gce#gce-coredns-performance-nodecache)\n  - [kube-dns-nodecache](https://k8s-testgrid.appspot.com/sig-network-gce#gci-gce-kube-dns-nodecache)\n\n\n### Graduation Criteria\n\nIn order to graduate to beta, we need:\n\n* Lock down the node-local-dns configmap so that Corefile cannot be modified directly.\n\n* Enablement of HA for NodeLocal DNSCache. With this support, the iptables rules management can be separated out to a different component. node-local-dns pod will accept multiple listen IP addresses as well.\n\n### Alternatives\n\nOne suggestion for HA that has come up is to list multiple nameservers in the client pods' /etc/resolv.conf - both node-local-dns IP as well as kube-dns service IP.\nThis is not recommended because the behavior is inconsistent depending on the client library. glibc 2.16+ and musl implementations send queries in parallel to both nameservers, so if we use both kube-dns IP as well as the link-local IP used by NodeLocal DNSCache, we could make the DNS query explosion problem worse. More queries means more conntrack entries and more DNATs.\nThis workaround could be viable for client implementations that do round-robin.\n\nRunning 2 daemonsets of node-local-dns using the same listenIP - 169.254.20.10 via SO_REUSEPORT option. Upgrades will be done one daemonset at a time.\n\n## Implementation History\n\nNodeLocal DNSCache was introduced in Kubernetes 1.13\n"
+  },
+  {
+    "id": "1a625767fb1cec3c48fd9cdb1f93a0db",
+    "title": "EndpointSlice API",
+    "authors": ["@freehan"],
+    "owningSig": "sig-network",
+    "participatingSigs": null,
+    "reviewers": ["@bowei", "@thockin", "@wojtek-t", "@johnbelamaric"],
+    "approvers": ["@bowei", "@thockin"],
+    "editor": "",
+    "creationDate": "2019-06-01",
+    "lastUpdated": "2019-06-01",
+    "status": "implementable",
+    "seeAlso": [
+      "https://docs.google.com/document/d/1sLJfolOeEVzK5oOviRmtHOHmke8qtteljQPaDUEukxY/edit#"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "# EndpointSlice API \n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goal](#goal)\n  - [Non-Goal](#non-goal)\n- [Proposal](#proposal)\n  - [EndpointSlice API](#endpointslice-api-1)\n  - [Mapping](#mapping)\n  - [EndpointMeta (Per EndpointSlice)](#endpointmeta-per-endpointslice)\n  - [Topology (Per Endpoint)](#topology-per-endpoint)\n  - [EndpointSlice Naming](#endpointslice-naming)\n- [Estimation](#estimation)\n- [Sample Case 1: 20,000 endpoints, 5,000 nodes](#sample-case-1-20000-endpoints-5000-nodes)\n  - [Service Creation/Deletion](#service-creationdeletion)\n  - [Single Endpoint Update](#single-endpoint-update)\n  - [Rolling Update](#rolling-update)\n- [Sample Case 2: 20 endpoints, 10 nodes](#sample-case-2-20-endpoints-10-nodes)\n  - [Service Creation/Deletion](#service-creationdeletion-1)\n  - [Single Endpoint Update](#single-endpoint-update-1)\n  - [Rolling Update](#rolling-update-1)\n- [Implementation](#implementation)\n  - [Requirements](#requirements)\n  - [EndpointSlice Controller](#endpointslice-controller)\n  - [Additional EndpointSlice Controllers](#additional-endpointslice-controllers)\n    - [Workflows](#workflows)\n  - [Kube-Proxy](#kube-proxy)\n  - [Endpoint Controller (classic)](#endpoint-controller-classic)\n- [Roll Out Plan](#roll-out-plan)\n- [Graduation Criteria](#graduation-criteria)\n  - [Splitting IP address type for better dual stack support](#splitting-ip-address-type-for-better-dual-stack-support)\n- [Alternatives](#alternatives)\n- [FAQ](#faq)\n\u003c!-- /toc --\u003e\n\n## Summary \n\nThis KEP was converted from the [original proposal doc][original-doc]. The current  [Core/V1 Endpoints API][v1-endpoints-api] comes with severe performance/scalability drawbacks affecting multiple components in the control-plane (apiserver, etcd, endpoints-controller, kube-proxy). \nThis doc proposes a new EndpointSlice API aiming to replace Core/V1 Endpoints API for most internal consumers, including kube-proxy.\nThe new EndpointSlice API aims to address existing problems as well as leaving room for future extension.\n\n\n## Motivation\n\nIn the current Endpoints API, one object instance contains all the individual endpoints of a service. Whenever a single pod in a service is added/updated/deleted, the whole Endpoints object (even when the other endpoints didn't change) is re-computed, written to storage (etcd) and sent to all watchers (e.g. kube-proxy). This leads to 2 major problems:\n\n- Storing multiple megabytes of endpoints puts strain on multiple parts of the system due to not having a paging system and a monolithic watch/storage design. [The max number of endpoints is bounded by the K8s storage layer (etcd)][max-object-size]], which has a hard limit on the size of a single object (1.5MB by default). That means attempts to write an object larger than the limit will be rejected. Additionally, there is a similar limitation in the watch path in Kubernetes apiserver. For a K8s service, if its Endpoints object is too large, endpoint updates will not be propagated to kube-proxy(s), and thus iptables/ipvs won’t be reprogrammed.\n- [Performance degradation in large k8s deployments.][perf-degrade] Not being able to efficiently read/update individual endpoint changes can lead to (e.g during rolling upgrade of a service) endpoints operations that are quadratic in the number of its elements. If one consider watches in the picture (there's one from each kube-proxy), the situation becomes even worse as the quadratic traffic gets multiplied further with number of watches (usually equal to #nodes in the cluster).\n\nThe new EndpointSlice API aims to address existing problems as well as leaving room for future extension.\n\n\n### Goal\n\n- Support tens of thousands of backend endpoints in a single service on cluster with thousands of nodes.\n- Move the API towards a general-purpose backend discovery API.\n- Leave room for foreseeable extension:\n  - Support multiple IPs per pod\n  - More endpoint states than Ready/NotReady\n  - Dynamic endpoint subsetting\n  \n### Non-Goal\n- Change functionality provided by K8s V1 Service API.\n- Provide better load balancing for K8s service backends.\n\n## Proposal\n\n### EndpointSlice API\nThe following new EndpointSlice API will be added to the `Discovery` API group.\n\n```\ntype EndpointSlice struct {\n    metav1.TypeMeta `json:\",inline\"`\n    // OwnerReferences should be set when the object is derived from a k8s\n    // object.\n    // The object labels may include the following keys:\n    // * kubernetes.io/service-name: the label value indicates the name of the\n    //   service from which the EndpointSlice is derived. EndpointSlices which\n    //   are not associated with a Service should not use this key.\n    // * endpointslice.kubernetes.io/managed-by: the label value represents a\n    //   unique name for the controller or application that manages this\n    //   EndpointSlice.\n    // +optional\n    metav1.ObjectMeta `json:\"metadata,omitempty\" protobuf:\"bytes,1,opt,name=metadata\"`\n    // addressType specifies the type of address carried by this EndpointSlice.\n    // All addresses in this slice must be the same type.\n    // Default is IP\n    // +optional\n    AddressType *AddressType `json:\"addressType\" protobuf:\"bytes,4,rep,name=addressType\"`\n    // endpoints is a list of unique endpoints in this slice. Each slice may\n    // include a maximum of 1000 endpoints.\n    // +listType=atomic\n    Endpoints []Endpoint `json:\"endpoints\" protobuf:\"bytes,2,rep,name=endpoints\"`\n    // ports specifies the list of network ports exposed by each endpoint in\n    // this slice. Each port must have a unique name. When ports is empty, it\n    // indicates that there are no defined ports. When a port is defined with a\n    // nil port value, it indicates \"all ports\". Each slice may include a\n    // maximum of 100 ports.\n    // +optional\n    // +listType=atomic\n    Ports []EndpointPort `json:\"ports\" protobuf:\"bytes,3,rep,name=ports\"`\n}\n\n// AddressType represents the type of address referred to by an endpoint.\ntype AddressType string\n\nconst (\n    // AddressTypeIP represents an IP Address.\n    // This address type has been deprecated and has been replaced by the IPv4\n    // and IPv6 adddress types. New resources with this address type will be\n    // considered invalid. This will be fully removed in 1.18.\n    // +deprecated\n    AddressTypeIP = AddressType(\"IP\")\n    // AddressTypeIPv4 represents an IPv4 Address.\n    AddressTypeIPv4 = AddressType(corev1.IPv4Protocol)\n    // AddressTypeIPv6 represents an IPv6 Address.\n    AddressTypeIPv6 = AddressType(corev1.IPv6Protocol)\n    // AddressTypeFQDN represents a FQDN.\n    AddressTypeFQDN = AddressType(\"FQDN\")\n)\n\n// Endpoint represents a single logical \"backend\" implementing a service.\ntype Endpoint struct {\n    // addresses of this endpoint. The contents of this field are interpreted\n    // according to the corresponding EndpointSlice addressType field. This\n    // allows for cases like dual-stack (IPv4 and IPv6) networking. Consumers\n    // (e.g. kube-proxy) must handle different types of addresses in the context\n    // of their own capabilities. This must contain at least one address but no\n    // more than 100.\n    // +listType=set\n    Addresses []string `json:\"addresses\" protobuf:\"bytes,1,rep,name=addresses\"`\n    // conditions contains information about the current status of the endpoint.\n    Conditions EndpointConditions `json:\"conditions,omitempty\" protobuf:\"bytes,2,opt,name=conditions\"`\n    // hostname of this endpoint. This field may be used by consumers of\n    // endpoints to distinguish endpoints from each other (e.g. in DNS names).\n    // Multiple endpoints which use the same hostname should be considered\n    // fungible (e.g. multiple A values in DNS). Must pass DNS Label (RFC 1123)\n    // validation.\n    // +optional\n    Hostname *string `json:\"hostname,omitempty\" protobuf:\"bytes,3,opt,name=hostname\"`\n    // targetRef is a reference to a Kubernetes object that represents this\n    // endpoint.\n    // +optional\n    TargetRef *v1.ObjectReference `json:\"targetRef,omitempty\" protobuf:\"bytes,4,opt,name=targetRef\"`\n    // topology contains arbitrary topology information associated with the\n    // endpoint. These key/value pairs must conform with the label format.\n    // https://kubernetes.io/docs/concepts/overview/working-with-objects/labels\n    // Topology may include a maximum of 16 key/value pairs. For endpoints\n    // backed by Kubernetes Pods, This may include, but is not limited to the\n    // following well known keys:\n    // * kubernetes.io/hostname: the value indicates the hostname of the node\n    //   where the endpoint is located. This should match the corresponding\n    //   node label.\n    // * topology.kubernetes.io/zone: the value indicates the zone where the\n    //   endpoint is located. This should match the corresponding node label.\n    // * topology.kubernetes.io/region: the value indicates the region where the\n    //   endpoint is located. This should match the corresponding node label.\n    // +optional\n    Topology map[string]string `json:\"topology,omitempty\" protobuf:\"bytes,5,opt,name=topology\"`\n}\n\n// EndpointConditions represents the current condition of an endpoint.\ntype EndpointConditions struct {\n    // ready indicates that this endpoint is prepared to receive traffic,\n    // according to whatever system is managing the endpoint. A nil value\n    // indicates an unknown state. In most cases consumers should interpret this\n    // unknown state as ready.\n    // +optional\n    Ready *bool `json:\"ready,omitempty\" protobuf:\"bytes,1,name=ready\"`\n}\n\n// EndpointPort represents a Port used by an EndpointSlice\ntype EndpointPort struct {\n    // The name of this port. All ports in an EndpointSlice must have a unique\n    // name. If the EndpointSlice is dervied from a Kubernetes service, this\n    // corresponds to the Service.ports[].name.\n    // Name must either be an empty string or pass IANA_SVC_NAME validation:\n    // * must be no more than 15 characters long\n    // * may contain only [-a-z0-9]\n    // * must contain at least one letter [a-z]\n    // * it must not start or end with a hyphen, nor contain adjacent hyphens\n    // Default is empty string.\n    Name *string `json:\"name,omitempty\" protobuf:\"bytes,1,name=name\"`\n    // The IP protocol for this port.\n    // Must be UDP, TCP, or SCTP.\n    // Default is TCP.\n    Protocol *v1.Protocol `json:\"protocol,omitempty\" protobuf:\"bytes,2,name=protocol\"`\n    // The application protocol for this port.\n    // +optional\n    AppProtocol *string `json:\"appProtocol,omitempty\" protobuf:\"bytes,3,name=appProtocol\"`\n    // The port number of the endpoint.\n    // If this is not specified, ports are not restricted and must be\n    // interpreted in the context of the specific consumer.\n    Port *int32 `json:\"port,omitempty\" protobuf:\"bytes,4,opt,name=port\"`\n}\n```\n\n### Mapping\n- 1 Service maps to N EndpointSlice objects.\n- Each EndpointSlice contains at most 100 endpoints by default (MaxEndpointThreshold: configurable via controller flag). \n- If a EndpointSlice is derived from K8s:\n  - The following label is added to identify corresponding service:  \n    - Key: kubernetes.io/service\n    - Value: ${service name}\n  - For EndpointSlice instances that are not derived from kubernetes Services, the above label must not be applied.\n  - The OwnerReferences of the EndpointSlice instances will be set to the corresponding service.\n- For backend pods with non-uniform named ports (e.g. a service port targets a named port. Backend pods have different port number with the same port name), this would amplify the number of EndpointSlice object depending on the number of backend groups with same ports.\n- EndpointSlice will be covered by resource quota. This is to limit the max number of EndpointSlice objects in one namespace. This would provide protection for k8s apiserver. For instance, a malicious user would not be able to DOS k8s API by creating services selecting all pods.\n\n### EndpointMeta (Per EndpointSlice)\nEndpointMeta contains metadata applying to all endpoints contained in the EndpointSlice.\n- **Endpoint Port**: The endpoint port number becomes optional in the EndpointSlice API while the port number field in core/v1 Endpoints API is required. This allows the API to support services with no port remapping or all port services.   \n\n### Topology (Per Endpoint)\nA new topology field (string to string map) is added to each endpoint. It can contain arbitrary topology information associated with the endpoint. If the EndpointSlice instance is derived from K8s service, the topology may contain following well known key:\n- **kubernetes.io/hostname**: the value indicates the hostname of the node where the endpoint is located. This should match the corresponding node label.\n- **topology.kubernetes.io/zone**: the value indicates the zone where the endpoint is located. This should match the corresponding node label.\n- **topology.kubernetes.io/region**: the value indicates the region where the endpoint is located. This should match the corresponding node label.\n\nIf the k8s service has topological keys specified, the corresponding node labels will be copied to endpoint topology. \n\n### EndpointSlice Naming\nUse `generateName` with service name as prefix:\n```\n${service name}-${random}\n```\n\n## Estimation\nThis section provides comparisons between Endpoints API and EndpointSlice API under 3 scenarios:\n- Service Creation/Deletion\n- Single Endpoint Update\n- Rolling Update\n\n```\nNumber of Backend Pod: P\nNumber of Node: N\nNumber of Endpoint Per EndpointSlice:B \n```\n\n## Sample Case 1: 20,000 endpoints, 5,000 nodes\n \n### Service Creation/Deletion\n\n\n|                          | Endpoints             | 100 Endpoints per EndpointSlice | 1 Endpoint per EndpointSlice |\n| # of writes              | O(1)                  | O(P/B)                          | O(P)                         |\n|                          | 1                     | 200                             | 20000                        |\n| Size of API object       | O(P)                  | O(B)                            | O(1)                         |\n|                          | 20k * const = ~2.0 MB | 100 * const = ~10 KB            | \u003c ~1KB                       |\n| # of watchers per object | O(N)                  | O(N)                            | O(N)                         |\n|                          | 5000                  | 5000                            | 5000                         |\n| # of total watch event   | O(N)                  | O(NP/B)                         | O(NP)                        |\n|                          | 5000                  | 5000 * 200 = 1,000,000          | 5000 * 20000 = 100,000,000   |\n| Total Bytes Transmitted  | O(PN)                 | O(PN)                           | O(PN)                        |\n|                          | 2.0MB * 5000 = 10GB   | 10KB * 5000 * 200 = 10GB        | ~10GB                        |\n\n### Single Endpoint Update\n\n|                          | Endpoints             | 100 Endpoints per EndpointSlice | 1 Endpoint per EndpointSlice |\n| # of writes              | O(1)                  | O(1)                            | O(1)                         |\n|                          | 1                     | 1                               | 1                            |\n| Size of API object       | O(P)                  | O(B)                            | O(1)                         |\n|                          | 20k * const = ~2.0 MB | 100 * const = ~10 KB            | \u003c ~1KB                       |\n| # of watchers per object | O(N)                  | O(N)                            | O(N)                         |\n|                          | 5000                  | 5000                            | 5000                         |\n| # of total watch event   | O(N)                  | O(N)                            | O(N)                         |\n|                          | 5000                  | 5000                            | 5000                         |\n| Total Bytes Transmitted  | O(PN)                 | O(BN)                           | O(N)                         |\n|                          | ~2.0MB * 5000 = 10GB  | ~10k * 5000 = 50MB              | ~1KB * 5000 = ~5MB           |\n\n\n### Rolling Update\n\n|                          | Endpoints                   | 100 Endpoints per EndpointSlice | 1 Endpoint per EndpointSlice |\n| # of writes              | O(P)                        | O(P)                            | O(P)                         |\n|                          | 20k                         | 20k                             | 20k                          |\n| Size of API object       | O(P)                        | O(B)                            | O(1)                         |\n|                          | 20k * const = ~2.0 MB       | 100 * const = ~10 KB            | \u003c ~1KB                       |\n| # of watchers per object | O(N)                        | O(N)                            | O(N)                         |\n|                          | 5000                        | 5000                            | 5000                         |\n| # of total watch event   | O(NP)                       | O(NP)                           | O(NP)                        |\n|                          | 5000 * 20k                  | 5000 * 20k                      | 5000 * 20k                   |\n| Total Bytes Transmitted  | O(P^2N)                     | O(NPB)                          | O(NP)                        |\n|                          | 2.0MB * 5000 * 20k = 200 TB | 10KB * 5000 * 20k = 1 TB        | ~1KB * 5000 * 20k = ~100 GB  |\n\n\n## Sample Case 2: 20 endpoints, 10 nodes\n \n### Service Creation/Deletion\n\n|                          | Endpoints             | 100 Endpoints per EndpointSlice | 1 Endpoint per EndpointSlice |\n| # of writes              | O(1)                  | O(P/B)                          | O(P)                         |\n|                          | 1                     | 1                               | 20                           |\n| Size of API object       | O(P)                  | O(B)                            | O(1)                         |\n|                          | ~1KB                  | ~1KB                            | ~1KB                         |\n| # of watchers per object | O(N)                  | O(N)                            | O(N)                         |\n|                          | 10                    | 10                              | 10                           |\n| # of total watch event   | O(N)                  | O(NP/B)                         | O(NP)                        |\n|                          | 1 * 10 = 10           | 1 * 10 = 10                     | 10 * 20 = 200                |\n| Total Bytes Transmitted  | O(PN)                 | O(PN)                           | O(PN)                        |\n|                          | ~1KB * 10 = 10KB      | ~1KB * 10 = 10KB                | ~1KB * 200 = 200KB           |\n\n### Single Endpoint Update\n\n|                          | Endpoints             | 100 Endpoints per EndpointSlice | 1 Endpoint per EndpointSlice |\n| # of writes              | O(1)                  | O(1)                            | O(1)                         |\n|                          | 1                     | 1                               | 1                            |\n| Size of API object       | O(P)                  | O(B)                            | O(1)                         |\n|                          | ~1KB                  | ~1KB                            | ~1KB                         |\n| # of watchers per object | O(N)                  | O(N)                            | O(N)                         |\n|                          | 10                    | 10                              | 10                           |\n| # of total watch event   | O(N)                  | O(N)                            | O(N)                         |\n|                          | 1                     | 1                               | 1                            |\n| Total Bytes Transmitted  | O(PN)                 | O(BN)                           | O(N)                         |\n|                          | ~1KB * 10 = 10KB      | ~1KB * 10 = 10KB                | ~1KB * 10 = 10KB             |\n\n\n### Rolling Update\n\n|                          | Endpoints                   | 100 Endpoints per EndpointSlice | 1 Endpoint per EndpointSlice |\n| # of writes              | O(P)                        | O(P)                            | O(P)                         |\n|                          | 20                          | 20                              | 20                           |\n| Size of API object       | O(P)                        | O(B)                            | O(1)                         |\n|                          | ~1KB                        | ~1KB                            | ~1KB                         |\n| # of watchers per object | O(N)                        | O(N)                            | O(N)                         |\n|                          | 10                          | 10                              | 10                           |\n| # of total watch event   | O(NP)                       | O(NP)                           | O(NP)                        |\n|                          | 10 * 20                     | 10 * 20                         | 10 * 20                      |\n| Total Bytes Transmitted  | O(P^2N)                     | O(NPB)                          | O(NP)                        |\n|                          | ~1KB * 10 * 20 = 200KB      | ~1KB * 10 * 20 = 200KB          | ~1KB * 10 * 20 = 200KB       |\n\n\n## Implementation\n\n### Requirements\n\n- **Persistence (Minimal Churn of Endpoints)**\n\nUpon service endpoint changes, the # of object writes and disruption to ongoing connections should be minimal. \n\n- **Handling Restarts \u0026 Failures**\n\nThe producer/consumer of EndpointSlice must be able to handle restarts and recreate state from scratch with minimal change to existing state.\n \n\n### EndpointSlice Controller\n\nA new EndpointSlice Controller will be added to `kube-controller-manager`. It will manage the lifecycle EndpointSlice instances derived from services.  \n```\nWatch: Service, Pod, Node ==\u003e Manage: EndpointSlice\n```\n\n### Additional EndpointSlice Controllers\n\nSince EndpointSlices were meant to be highly extensible, it's important to\nensure that they can be managed by other controllers without being deleted or\nmodified by the primary EndpointSlice Controller. To achieve that, we propose\nadding a `endpointslice.kubernetes.io/managed-by` label. The EndpointSlice\ncontroller will set a value of `endpointslice-controller` on each EndpointSlice\nit manages. It will not modify any EndpointSlices without that label value.\n\nIn the alpha release of EndpointSlices in 1.16, this label did not exist and all\nEndpointSlices associated with a Service that had a selector specified were\nmanaged by the EndpointSlice Controller. To add support for this label in 1.17,\na temporary `endpointslice.kubernetes.io/managed-by-setup` annotation on the\nService will be used to provide a seamless upgrade. In 1.17, the EndpointSlice\ncontroller will claim each EndpointSlice without the corresponding label and\nannotation set, setting those values to claim ownership. In 1.18, the annotation\non the Service can safely be removed.\n\n#### Workflows\nOn Service Create/Update/Delete:\n- `syncService(svc)`\n\nOn Pod Create/Update/Delete: \n- Reverse lookup relevant services\n- For each relevant service, \n  - `syncService(svc)`\n\n`syncService(svc)`:\n- Look up selected backend pods\n- Look up existing EndpointSlices for the service `svc`.\n- Calculate difference between wanted state and current state.  \n- Perform reconciliation with minimized changes.\n\n### Kube-Proxy\n\nKube-proxy will be modified to consume EndpointSlice instances besides Endpoints resource. A flag will be added to kube-proxy to toggle the mode. \n\n```\nWatch: Service, EndpointSlice ==\u003e Manage: iptables, ipvs, etc\n```\n- Merge multiple EndpointSlice into an aggregated list.\n- Reuse the existing processing logic \n\n### Endpoint Controller (classic)\n\nIn order to ensure backward compatibility for external consumer of the core/v1 Endpoints API, the existing K8s endpoint controller will keep running until the API is EOL. The following limitations will apply:\n- Starting from EndpointSlice beta: If # of endpoints in one Endpoints object exceed 1000, generate a warning event to the object. \n- Starting from EndpointSlice GA: Only include up to 1000 endpoints in one Endpoints Object and throw events.\n\n## Roll Out Plan\n\n| K8s Version | State | OSS Controllers                                                            | Internal Consumer (Kube-proxy) |\n| 1.16        | Alpha | EndpointSliceController (Alpha) EndpointController (GA with normal event)  | Endpoints                      |\n| 1.17        | Beta  | EndpointSliceController (Beta)  EndpointController (GA with warning event) | EndpointSlice                  |\n| 1.19+       | GA    | EndpointSliceController (GA)    EndpointController (GA with limitation)    | EndpointSlice                  |\n\n\n## Graduation Criteria\n\nIn order to graduate to beta, we will:\n\n- Kube-proxy switch to consume EndpointSlice API. (Already done in Alpha)\n- Verify performance/scalability via testing. (Scale tested to 50k endpoints in\n  4k node cluster)\n- Get performance fixes identified in scale testing merged.\n- Split `IP` address type into new `IPv4` and `IPv6` address types.\n- Implement dual-stack EndpointSlice support in kube-proxy.\n- Implement e2e tests that ensure both Endpoints and EndpointSlices are tested.\n- Add support for `endpointslice.kubernetes.io/managed-by` label.\n- Add FQDN addressType.\n- Add support for optional appProtocol field on `EndpointPort`.\n\n### Splitting IP address type for better dual stack support\n\nAlthough the initial vision for the `IP` address type was to be inclusive of\nboth IPv4 and IPv6 addresses, that ended up complicating workflows in consumers\nlike kube-proxy. In that case, and we anticipate many more, the consumer is only\ninterested in a specific IP family for an endpoint. Both Endpoints and Services\nhave moved toward using different resources per IP family. It only makes sense\nto mirror that behavior with EndpointSlices.\n\nWith that in mind, the proposed changes for beta will involve the following:\n\n1. Add 2 additional address types: `IPv4` and `IPv6`.\n2. Update the EndpointSlice controller to only create EndpointSlices with these\naddress types.\n3. Deprecate the `IP` address type, making it invalid for new EndpointSlices in\n1.17 before becoming fully invalid in 1.18.\n\n## Alternatives\n\n1. increase the etcd size limits\n2. endpoints controller batches / rate limits changes\n3. apiserver batches / rate-limits watch notifications\n4. apimachinery to support object level pagination\n\n## FAQ\n\n- #### Why not pursue the alternatives?\n\nIn order to fulfill the goal of this proposal, without redesigning the Core/V1 Endpoints API, all items listed in the alternatives section are required. Item #1 increase maximum endpoints limitation by increasing the object size limit. This may bring other performance/scalability implications. Item #2 and #3 can reduce transmission overhead but sacrificed endpoint update latency. Item #4 can further reduce transmission overhead, however it is a big change to the existing API machinery. \n\nIn summary, each of the items can only achieve incremental gain to some extent. Compared to this proposal, the combined effort would be equal or more while achieving less performance improvements. \n\nIn addition, the EndpointSlice API is capable to express endpoint subsetting, which is the natural next step for improving k8s service endpoint scalability.      \n\n- #### Why only include up to 100 endpoints in one EndpointSlice object? Why not 1 endpoint? Why not 1000 endpoints?\n\nBased on the data collected from user clusters, vast majority (\u003e 99%) of the k8s services have less than 100 endpoints. For small services, EndpointSlice API will make no difference. If the MaxEndpointThreshold is too small (e.g. 1 endpoint per EndpointSlice), controller loses capability to batch updates, hence causing worse write amplification on service creation/deletion and scale up/down. Etcd write RPS is significant limiting factor.\n\n- #### Why do we have a condition struct for each endpoint? \n\nThe current Endpoints API only includes a boolean state (Ready vs. NotReady) on individual endpoint. However, according to pod life cycle, there are more states (e.g. Graceful Termination, ContainerReary). In order to represent additional states other than Ready/NotReady,  a status structure is included for each endpoint. More condition types can be added in the future without compatibility disruptions. As more conditions are added, different consumer (e.g. different kube-proxy implementations) will have the option to evaluate the additional conditions. \n\n- #### Why not use a CRD?\n\n**1. Protobuf is more efficient**\nCurrently CRDs don't support protobuf. In our testing, a protobuf watch is\napproximately 5x faster than a JSON watch. We used pprof to profile 2 versions\nof kube-proxy using EndpointSlices and running on 2 different nodes in a 150\nnode cluster as it scaled up to 15k endpoints. Over the 15 minute window,\nkube-proxy with JSON used 17% more CPU time, with the difference in\n`StreamWatcher.receive` accounting for all of that. With protobuf enabled, that\nfunction took 1/5th the time of the JSON implementation.\n\n**2. Validation is too complex**\nValidation of addresses relies on addressType, something that would be\ndifficult, maybe impossible, to recreate with OpenAPI validations. Additionally,\nthere are a number of validations currently in use that are able to reuse the\nsame validations used elsewhere for Services or Endpoints such as\n`IsDNS1123Label` and `IsValidIP`. Although these could be recreated with OpenAPI\nvalidations, the error messages would not be as helpful and we would lose the\nconsistency in messaging from the related resources.\n\n**3. EndpointSlices are required for the API Server to be accessible**\nIn an interesting race condition, the API Server needs to be available before\nmuch can happen. With both Endpoints and EndpointSlices, that means that it\nneeds to manage the references to itself by creating these resources on startup.\nSince this makes EndpointSlice core enough to be a dependency of API Server, it\nwould have to exist in every cluster. If EndpointSlices were a CRD, the CRD\nwould also have to be installed by the API Server, making a CRD a core\ndependency of Kubernetes. Additionally, any components like kube-proxy that\ndepend on EndpointSlices would break if the CRD hadn't been installed before\nthey started up.\n\n\n[original-doc]: https://docs.google.com/document/d/1sLJfolOeEVzK5oOviRmtHOHmke8qtteljQPaDUEukxY/edit#\n[v1-endpoints-api]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.14/#endpoints-v1-core\n[max-object-size]: https://github.com/kubernetes/kubernetes/issues/73324\n[perf-degrade]: https://github.com/kubernetes/community/blob/master/sig-scalability/blogs/k8s-services-scalability-issues.md#endpoints-traffic-is-quadratic-in-the-number-of-endpoints\n"
+  },
+  {
+    "id": "cc57bde3ea63a8ccc41954247e9a38b4",
+    "title": "Kube-Proxy ComponentConfig graduation",
+    "authors": ["@rosti"],
+    "owningSig": "sig-network",
+    "participatingSigs": ["sig-cluster-lifecycle", "sig-api-machinery", "wg-component-standard"],
+    "reviewers": ["@luxas", "@mtaufen", "@sttts"],
+    "approvers": ["@thockin"],
+    "editor": "@rosti",
+    "creationDate": "2019-06-13",
+    "lastUpdated": "",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# kube-proxy component config graduation proposal\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Re-encapsulate mode specific options](#re-encapsulate-mode-specific-options)\n    - [Example](#example)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n**ACTION REQUIRED:** In order to merge code into a release, there must be an issue in [kubernetes/enhancements] referencing this KEP and targeting a release milestone **before [Enhancement Freeze](https://github.com/kubernetes/sig-release/tree/master/releases)\nof the targeted release**.\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e., [kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be updated for the enhancement to be released.\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [X] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nThis document is intended to propose a process and desired goals by which kube-proxy's component configuration is to be graduated to beta.\n\n## Motivation\n\nkube-proxy is a component, that is present in almost all Kubernetes clusters in existence.\nHistorically speaking, kube-proxy's configuration was supplied by a set of command line flags. Over time, the number of flags grew and they became unwieldy to use and support. Thus, kube-proxy gained component config.\nInitially this was just a large flat object, that was representing the command line flags. However, over time new features were added to it, all while staying as v1alpha1.\n\nThis resulted in a configuration format, that had various different options grouped together in ways, that made them hard to specify and understand. For example:\n\n- Instance local options (such as host name override, bind address, etc.) are in the same flat object as shared between instances options (such as the cluster CIDR, config sync period, etc.).\n- Platform specific options are mixed together. For example, the IPTables rule sync fields are used by the Windows HNS backend for the same purpose.\n- Again, the IPTables rule sync options are used for the Linux legacy user mode proxy, but not for the IPVS mode (where a set of identical options exist, despite the fact, that it too uses some other fields, designed for IPTables).\n\nClearly, this made the configuration both hard to use and to maintain. Therefore, a plan to restructure and stabilize the config format is needed.\n\n### Goals\n\n- To cleanup the existing config format.\n- To provide config structure, that is easier for users to understand and use.\n- To distinguish between instance local and shared settings.\n- To allow for the persistence of settings for different platforms (such as Linux and Windows) in a manner that reduces confusion and the possibility of an error.\n- To allow for easier introduction of new proxy backends.\n- To provide users with flexibility, especially with regards to the config source.\n\n### Non-Goals\n\n- To change or implement additional features in kube-proxy.\n- To deal with graduation of any other component of kube-proxy, other than its configuration.\n- To remove most or even all of the command line flags, that have corresponding component config options.\n\n## Proposal\n\nThe idea is to conduct the process of graduation to beta in small steps in the span of at least one Kubernetes release cycle. This will be done by creating one or more alpha versions of the config with the last alpha version being copied as v1beta1 after the community is happy with it.\nEach of the sub-sections below can result in a separate alpha version release, although it will be better for users to have no more than a couple of alpha versions past v1alpha1.\nAfter each alpha version release, the community will gather around for new ideas on how to proceed in the graduation process. If there are viable proposals, this document is updated with an appropriate section(s) below and the new changes are introduced in the form of new alpha version(s).\nThe proposed process is similar to the already successfully used one for kubeadm.\n\n### Re-encapsulate mode specific options\n\nThe current state of the config has proven that:\n- Some options are deemed as mode specific, but are in fact shared between all modes.\n- Some options are placed directly into KubeProxyConfiguration, but are in fact mode specific ones.\n- There are options that are shared between some (but not all) modes. Specific features of the underlying implementation are common and this happens only within the boundaries of the platform (iptables and ipvs modes for example).\n- Although legacy Linux and Windows user mode proxies are separate code bases, they have a common set of options.\n\nWith that in mind, the following measures are proposed:\n- Mode specific structs are consolidated to not use fields from other mode specific structs.\n- Introduce a single combined legacy user mode proxy struct for both Linux and Windows backends.\n\n#### Example\n\n```yaml\ncommonSetting1: ...\ncommonSetting2: ...\n...\nmodeA: ...\nmodeB: ...\nmodeC: ...\n```\n\n### Risks and Mitigations\n\nSo far, the following risks have been identified:\n- Deviation of the implementation guidelines and bad planning may have the undesired effect of producing bad alpha versions.\n- Bad alpha versions will need good alpha versions to fix them. This will create too many iterations over the API and users may get confused.\n- New and redesigned kube-proxy API versions may cause confusion among users who are used to the v1alpha1 relatively flat, single document design. In particular, multiple YAML documents and structured (as opposed to flat) objects can create confusion as to what option is placed where.\n\nThe mitigations to those risks:\n- Strict following of the proposals in this document and planning ahead for a release and config cycle.\n- Support reading from the last couple of API versions released. When the beta version is released, support the last alpha version for one or two release cycles after that.\n- Documentation on the new APIs and how to migrate to them.\n- Provide optional migration tool for the APIs.\n\n## Design Details\n\n### Test Plan\n\nExisting test cases throughout the kube-proxy code base should be adapted to use the latest config version.\nIf required, new test cases should also be created.\n\n### Graduation Criteria\n\nThe config should be considered graduated to beta if it:\n- is well structured with clear boundaries between different proxy mode settings.\n- allows for easy multi-platform use with less probability of an error.\n- allows for easy distinguishment between instance local and shared settings.\n- is well covered by tests.\n- is well documented. Especially with regards of migrating to it from older versions.\n"
+  },
+  {
+    "id": "487b738d2dcba97dd30322f96537342f",
+    "title": "graduate-ipv6-to-beta",
+    "authors": ["@aojea"],
+    "owningSig": "sig-network",
+    "participatingSigs": null,
+    "reviewers": ["@bentheelder", "@andrewsykim", "@khenidak"],
+    "approvers": ["@lachie83", "@thockin"],
+    "editor": "TBD",
+    "creationDate": "2019-07-14",
+    "lastUpdated": "2019-07-27",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Graduate IPv6 to beta\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n  - [User Stories](#user-stories)\n- [Proposal](#proposal)\n- [Design](#design)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nSupport for IPv6-only clusters was added in Kubernetes 1.9 as an alpha feature, allowing full Kubernetes capabilities using IPv6 networking instead of IPv4 networking. It also included support for Kubernetes IPv6 cluster deployments using kubeadm and support for the iptables kube-proxy backend using ip6tables. With version 1.13 the Kubernetes default DNS server changed to CoreDNS which has full IPv6 support.\n\n## Motivation\n\nIPv6 adoption is ramping up with the advances in IoT space and explosion in number of mobile devices, this adoption will continue to grow as we can observe at the [Google IPv6 adoption statistics](https://www.google.com/intl/en/ipv6/statistics.html).\n\nThere are cloud providers that already support IPv6 and can deploy kubernetes IPv6 only clusters, a new CI with e2e testing is running all conformance tests using [kind](https://kind.sigs.k8s.io/) and dual stack support will be added during this release cycle (1.16), that will facilitate the migration from IPv4 to IPv6.\n\nTherefore, we would like to graduate IPv6 support from Alpha to Beta.\n\n### Goals\n\n* Promote IPv6 to beta.\n\n### Non-Goals\n\n* IPv6 is NOT Dual-stack.\n\n### User Stories\n\n* A user can deploy, operate and use an IPv6-only kubernetes cluster.\n\n## Proposal\n\nThe IPv6 support introduced in 1.9 allowed full Kubernetes capabilities using IPv6 networking instead of IPv4 networking. For beta we will provide enough e2e testing to guarantee it:\n\n* CI jobs with e2e and conformance testing will be implemented, using a virtual test environment with Kind and at least in one Cloud Provider.\n\n* The CI jobs will publish the results on testgrid.\n\n* The CI jobs running e2e conformance tests will be promoted to release-blokcing following the standard process, proving that they are stable and contacting SIG-release.\n\n* To ensure there are no regression, an IPv6 e2e job will be added as a presubmit job.\n\n## Design\n\n### Test Plan\n\nThere is no need to develop new tests, IPv6 only clusters should pass the same e2e tests that IPv4 only clusters, guaranteeing the feature parity.\n\n- Run E2E tests on an IPv6 only kubernetes cluster using kind.\n- Run E2E tests on an IPv6 only kubernetes on at least one cloud provider.\n\n### Graduation Criteria\n- [x] It has IPv4 feature parity\n- [x] It has CI using a kubernetes testing environment\n- [ ] It has CI using at least one Cloud Provider\n- [x] It has passed all e2e conformance tests\n- [x] It is documented\n\n## Implementation History\n\n- [IPv6 Support was introduced as alpha in kubernetes 1.9](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.9.md#ipv6)\n- [IPv6 Support enhancement request](https://github.com/kubernetes/enhancements/issues/508)\n- [IPv6 implementation tracking issue](https://github.com/kubernetes/kubernetes/issues/1443)\n- [IPv6 CI](https://testgrid.k8s.io/conformance-kind#kind%20(IPv6),%20master%20(dev)) \n- [Kind IPv6 support](https://github.com/kubernetes-sigs/kind/pull/636)\n"
+  },
+  {
+    "id": "d111447a0d17d51e42630da0ac3473f4",
+    "title": "Move ExternalDNS out of Kubernetes incubator",
+    "authors": ["@njuettner"],
+    "owningSig": "sig-network",
+    "participatingSigs": null,
+    "reviewers": null,
+    "approvers": null,
+    "editor": "",
+    "creationDate": "",
+    "lastUpdated": "",
+    "status": "",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Move ExternalDNS out of Kubernetes incubator\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n- [Proposal](#proposal)\n- [Details](#details)\n  - [Graduation Criteria](#graduation-criteria)\n    - [Maintainers](#maintainers)\n  - [Release process, artifacts](#release-process-artifacts)\n  - [Risks and Mitigations](#risks-and-mitigations)\n\u003c!-- /toc --\u003e\n\n## Summary\n\n[ExternalDNS](https://github.com/kubernetes-incubator/external-dns) is a project that synchronizes Kubernetes’ Services, Ingresses and other Kubernetes resources to DNS backends for several DNS providers.\n\nThe projects was started as a Kubernetes Incubator project in February 2017 and being the Kubernetes incubation initiative officially over, the maintainers want to propose the project to be moved to the kubernetes GitHub organization or to kubernetes-sigs, under the sponsorship of sig-network.\n\n## Motivation\n\nExternalDNS started as a community project with the goal of unifying several existing projects that were trying to solve the same problem: create DNS records for Kubernetes resources on several DNS backends.\n\nWhen the project was proposed (see the [original discussion](https://github.com/kubernetes/kubernetes/issues/28525#issuecomment-270766227)), there were at least 3 existing implementations of the same functionality:\n\n* Mate - [https://github.com/linki/mate](https://github.com/linki/mate)\n\n* DNS-controller from kops - [https://github.com/kubernetes/kops/tree/master/dns-controller](https://github.com/kubernetes/kops/tree/master/dns-controller)\n\n* Route53-kubernetes - [https://github.com/wearemolecule/route53-kubernetes](https://github.com/wearemolecule/route53-kubernetes)\n\nExternalDNS’ goal from the beginning was to provide an officially supported solution to those problems.\n\nAfter two years of development, the project is still in the kubernetes-incubator.\n\nThe incubation has been officially discontinued and to quote @thockin \"Incubator projects should either become real projects in Kubernetes, shut themselves down, or move elsewhere\" (see original thread [here](https://groups.google.com/forum/#!topic/kubernetes-sig-network/fvpDC_nxtEM)).\n\nThis KEP proposes to move ExternalDNS to the main Kubernetes organization or kubernetes-sigs. The \"Proposal\" section details the reasons behind it.\n\n### Goals\n\nThe only goal of this KEP is to establish consensus regarding the future of the ExternalDNS project and determine where it belongs.\n\n## Proposal\n\nThis KEP is about moving External DNS out of the Kubernetes incubator. This section will cover the reasons why External DNS is useful and what the community would miss in case the project would be discontinued or moved under another organization.\n\nExternal DNS...\n\n* Is the de facto solution to create DNS records for several Kubernetes resources.\n\n* Is a vital component to achieve an experience close to a PaaS that many Kubernetes users try to replicate on top of Kubernetes, by allowing to automatically create DNS records for web applications.\n\n* Supports already 18 different DNS providers including all major public clouds (AWS, Azure, GCP).\n\nGiven that the kubernetes-incubator organization will eventually be shut down, the possible alternatives to moving to be an official Kubernetes project are the following:\n\n* Shut down the project\n\n* Move the project elsewhere\n\nWe believe that those alternatives would result in a worse outcome for the community compared to moving the project to the any of the other official Kubernetes organizations.\nIn fact, shutting down ExternalDNS can cause:\n\n* The community to rebuild the same solution as already happened multiple times before the project was launched. Currently ExternalDNS is easy to be found, referenced in many articles/tutorials and for that reason not exposed to that risk.\n\n* Existing users of the projects to be left without a future proof working solution.\n\nMoving the ExternalDNS project outside of Kubernetes projects would cause:\n\n* Problems (re-)establishing user trust which could eventually lead to fragmentation and duplication.\n\n* It would be hard to establish in which organization the project should be moved to. The most natural would be Zalando’s organization, being the company that put most of the work on the project. While it is possible to assume Zalando’s commitment to open-source, that would be a strategic mistake for the project community and for the Kubernetes ecosystem due to the obvious lack of neutrality.\n\n* Lack of resources to test, lack of issue management via automation.\n\nFor those reasons, we propose to move ExternalDNS out of the Kubernetes incubator, to live either under the kubernetes or kubernetes-sigs organization to keep being a vital part of the Kubernetes ecosystem.\n\n\n## Details\n\n### Graduation Criteria\n\nExternalDNS is a two years old project widely used in production by many companies. The implementation for the three major cloud providers (AWS, Azure, GCP) is stable, not changing its logic and the project is being used in production by many company using Kubernetes.\n\nWe have evidence that many companies are using ExternalDNS in production, but it is out of scope for this proposal to collect a comprehensive list of companies.\n\nThe project was quoted by a number of tutorials on the web, including the [official tutorials from AWS](https://aws.amazon.com/blogs/opensource/unified-service-discovery-ecs-kubernetes/).\n\nExternalDNS can’t be consider to be \"done\": while the core functionality has been implemented, there is lack of integration testing and structural changes that are needed.\n\nThose are identified in the project roadmap, which is roughly made of the following items:\n\n* Decoupling of the providers\n\n    * Implementation proposal\n\n    * Development\n\n* Bug fixing and performance optimization (i.e. rate limiting on cloud providers)\n\n* Integration testing suite, to be implemented at least for the \"stable\" providers\n\n* ExternalDNS build/artifact has to use an official k8s.io release image\n\nFor those reasons, we consider ExternalDNS to be in Beta state as a project. We believe that once the items mentioned above will be implemented, the project can reach a declared GA status.\n\nThere are a number of other factors that need to be covered to fully describe the state of the project, including who are the maintainers, the way we release and manage the project and so on.\n\n#### Maintainers\n\nThe project has the following maintainers:\n\n* hjacobs\n\n* Raffo\n\n* linki\n\n* njuettner\n\nThe list of maintainers shrunk over time as people moved out of the original development team (all the team members were working at Zalando at the time of project creation) and the project required less work.\n\nThe high number of providers contributed to the project pose a maintainability challenge: it is hard to bring the providers forward in terms of functionalities or even test them. The maintainers believe that the plan to transform the current Provider interface from a Go interface to an API will allow for enough decoupling and to hand over the maintenance of those plugins to the contributors themselves, see the risk and mitigations section for further details.\n\n### Release process, artifacts\n\nThe project uses the free quota of TravisCI to run tests for the project.\n\nThe release pipeline for the project is currently fully owned by Zalando. It runs on the internal system of the company (closed source) which external maintainers/users can’t access and that pushes images to the publicly accessible docker registry available at the URL `registry.opensource.zalan.do`.\n\nThe docker registry service is provided as best effort with no sort of SLA and the maintainers team openly suggests the users to build and maintain their own docker image based on the provided Dockerfiles.\n\nProviding a vanity URL for the docker images was consider a non goal till now, but the community seems to be wanting official images from a GCR domain, similarly to what is available for other parts of official Kubernetes projects.\n\nExternalDNS does not follow a specific release cycle. Releases are made often when there are major contributions (i.e. new providers) or important bug fixes. That said, the master is considered stable and can be used as well to build images.\n\n### Risks and Mitigations\n\nThe following are risks that were identified:\n\n* Low number of maintainers: we are currently facing issues keeping up with the number of pull requests and issues giving the low number of maintainers. The list of maintainers already shrunk from 8 maintainers to 4.\n\n* Issues maintaining community contributed providers: we often lack access to external providers (i.e. InfoBlox, etc.) and this means that we cannot verify the implementations and/or run regression tests that go beyond unit testing.\n\n* Somewhat low quality of releases due to lack of integration testing.\n\nWe think that the following actions will constitute appropriate mitigations:\n\n* Decoupling the providers via an API will allow us to resolve the problem of the providers. Being the project already more than 2 years old and given that there are 18 providers implemented, we possess enough informations to define an API that we can be stable in a short timeframe. Once this is stable, the problem of testing the providers can be deferred to be a provider’s responsibility. This will also reduce the scope of External DNS core code, which means that there will be no need for a further increase of the maintaining team.\n\n* We added integration testing for the main cloud providers to the roadmap for the 1.0 release to make sure that we cover the mostly used ones. We believe that this item should be tackled independently from the decoupling of providers as it would be capable of generating value independently from the result of the decoupling efforts.\n\n* With the move to the Kubernetes incubation, we hope that we will be able to access the testing resources of the Kubernetes project. In this way, we hope to decouple the project from the dependency on Zalando’s internal CI tool. This will help open up the possibility to increase the visibility on the project from external contributors, which currently would be blocked by the lack of access to the software used for the whole release pipeline.\n"
+  },
+  {
+    "id": "b28ef9a2446cb4046d2fa1345afe2f6a",
+    "title": "Remove knowledge of pod cluster CIDR from iptables rules",
+    "authors": ["@satyasm"],
+    "owningSig": "sig-network",
+    "participatingSigs": null,
+    "reviewers": [
+      "@thockin",
+      "@caseydavenport",
+      "@mikespreitzer",
+      "@aojea",
+      "@fasaxc",
+      "@squeed",
+      "@bowei",
+      "@dcbw",
+      "@darwinship"
+    ],
+    "approvers": ["@thockin"],
+    "editor": "TBD",
+    "creationDate": "2019-11-04",
+    "lastUpdated": "2019-11-27",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Removing Knowledge of pod cluster CIDR from iptables rules\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Drawbacks [optional]](#drawbacks-optional)\n- [Alternatives [optional]](#alternatives-optional)\n  - [Multiple cluster CIDR rules](#multiple-cluster-cidr-rules)\n  - [ip-masq-agent like behavior](#ip-masq-agent-like-behavior)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThe iptables implementation of kube-proxy today references the cluster CIDR for pods in three places for the following reasons.\n\n   1. [Masquerade off cluster traffic to services by node IP](https://github.com/kubernetes/kubernetes/blob/v1.17.0/pkg/proxy/iptables/proxier.go#L965-L970)\n   2. [Redirecting pods traffic to external loadbalancer VIP to cluster IP](https://github.com/kubernetes/kubernetes/blob/v1.17.0/pkg/proxy/iptables/proxier.go#L1327-L1339)\n   3. [Accepting traffic after first packet, after being accepted by kubernetes rules](https://github.com/kubernetes/kubernetes/blob/v1.17.0/pkg/proxy/iptables/proxier.go#L1468-L1490)\n\nIn addition, the ipvs implementation also references it in two places for similar purposes\n\n   1. [Masquerade off cluster traffic to services by node IP](https://github.com/kubernetes/kubernetes/blob/master/pkg/proxy/ipvs/proxier.go#L1649-L1654)\n   2. [Accepting traffic after first packet, after being accepted by kubernetes](https://github.com/kubernetes/kubernetes/blob/master/pkg/proxy/ipvs/proxier.go#L1726-L1745)\n\nThis enhancement proposes ways to achieve similar goals without tracking the pod cluster CIDR to do so.\n\n## Motivation\n\nThe idea that makes kubernetes networking model unique and powerful is the concept of each pod having its own IP, \nwith all the pod IPs being natively routable within the cluster. The service chains in iptable rules depend on this \ncapability by assuming that they can treat all the endpoints of a cluster as being equivalent and load balance service \ntraffic across all the endpoints, by just translating destination to the pod IP address.\n\nWhile this is powerful, it also means pod IP addresses are in many cases the constraining resource for cluster creation\nand scale. It would be valuable for implementations to have different strategies for managing pod IP addresses that can\nadapt to different environment needs.\n\nSome examples of use cases:\n\n   * Creating a cluster out of many disjoint ranges instead of a single range.\n   * Expanding a cluster with more disjoint ranges after initial creation.\n\nNot having to depend on the cluster pod CIDR for routing service traffic would effectively de-couple pod IP management\nand allocation strategies from service management and routing. Which in turn would mean that it would be far cheaper \nto evolve the IP allocation schemes while sharing the same service implementation, thus significantly lowering the bar\nfor adoption of alternate schemes.\n\nAlternate implementations that don’t use iptables could also adopt this same reasoning to not have to track the cluster\nCIDR for routing cluster traffic.\n\n### Goals\n\n   * Not having to depend on the cluster pod CIDR for iptable rules and cluster traffic routing.\n\n### Non-Goals\n\n   * Providing alternate models of IP allocation schemes for pod CIDR.\n   * Enhancing current allocators to handle disjoint ranges.\n   * Enhancing current allocators to add additional ranges after cluster creation.\n   * Changing current assumptions around having a single pod CIDR per node.\n\n## Proposal\n\nAs stated above, the goal is to re-implement the functionality called out in the summary, but in a \nway that does not depend on a pod cluster CIDR. The essence of the proposal is that for the \nfirst two cases in iptables implementation and first case in ipvs, we can replace the `-s proxier.clusterCIDR` with \nsome notion of node local pod traffic.\n\nThe core logic in these cases is “how to determine” cluster originated traffic from non-cluster originated ones. \nThe proposal is that tracking pod traffic generated from within the node is sufficient to determine cluster originated \ntraffic. For the first two use cases in iptables and first use case in ipvs, we provide alternatives to using \nproxier.clusterCIDR in one of the following ways to determine cluster originated traffic\n\n   1. `-s node.podCIDR` (where node podCIDR is used for allocating pod IPs within the node)\n   2. `--in-interface prefix+` (where all pod interfaces start with same prefix,\n      or where all pod traffic appears to come from a single bridge or other interface)\n   3. `-m physdev --physdev-is-in` (for kubenet if we don’t want to depend on node podCIDR)\n\nNote the above are equivalent definitions, when considering only pod traffic originating from within the node.\n\nGiven that this kep only addresses usage of the cluster CIDR (for pods), and that pods with hostNetwork are not \nimpacted by this, the assumption is that hostNetwork pod behavior will continue to work as is.\n\nFor the last use case, note above, in iptables and ipvs, the proposal is to drop the reference to the cluster CIDR.\n\nThe reasoning behind why this works are as follows.\n\n### iptables - masquerade off cluster traffic to services by node IP\n\nThe [rule here currently](\n  https://github.com/kubernetes/kubernetes/blob/v1.17.0/pkg/proxy/iptables/proxier.go#L965-L970\n) looks as follows\n\n```go\n// This masquerades off-cluster traffic to a service VIP.  The idea\n// is that you can establish a static route for your Service range,\n// routing to any node, and that node will bridge into the Service\n// for you.  Since that might bounce off-node, we masquerade here.\n// If/when we support \"Local\" policy for VIPs, we should update this.\nwriteLine(proxier.natRules, append(args, \"! -s\", proxier.clusterCIDR, \"-j\", string(KubeMarkMasqChain))...)\n```\n\nThe logic is that if the source IP is not part of the cluster CIDR range,\nthen it must have originated from outside the cluster. Hence we add a rule to masquerade by\nthe node IP so that we can send traffic to any pod within the cluster.\n\nOne key insight when thinking about this data path though is the fact that the iptable rules run\nat _every_ node boundary. So when a pod sends a traffic to a service IP, it gets translated to\none of the pod IPs _before_ it leaves the node at the node boundary. So it's highly unlikely to \nreceive traffic at a node, whose destination is the service cluster IP, that is initiated by pods\nwithin the cluster, but not scheduled within that node.\n\nGoing by the above reasoning, if we receive traffic destined to a service whose source is not within the node \ngenerated pod traffic, we can say with very high confidence that the traffic originated from outside the cluster. \nSo we can rewrite the rule in terms of just the pod identity within the node (node CIDR, interface prefix or bridge).\nThis would be the simplest change with respect to re-writing the rule without any assumptions on how pod \nnetworking is setup.\n\n### iptables - redirecting pod traffic to external loadbalancer VIP to cluster IP\n\nThe [rule here currently](\n  https://github.com/kubernetes/kubernetes/blob/v1.17.0/pkg/proxy/iptables/proxier.go#L1327-L1339\n) looks as follows\n\n```go\n// First rule in the chain redirects all pod -\u003e external VIP traffic to the\n// Service's ClusterIP instead. This happens whether or not we have local\n// endpoints; only if clusterCIDR is specified\nif len(proxier.clusterCIDR) \u003e 0 {\n  args = append(args[:0],\n    \"-A\", string(svcXlbChain),\n    \"-m\", \"comment\", \"--comment\",\n    `\"Redirect pods trying to reach external loadbalancer VIP to clusterIP\"`,\n    \"-s\", proxier.clusterCIDR,\n    \"-j\", string(svcChain),\n  )\n  writeLine(proxier.natRules, args...)\n}\n```\n\nThe logic here is that if the source IP is part of cluster CIDR and we detect that is being\nsent to a load balancer IP for a service, we short circuit it by jumping directly to the\nservice chain instead of having the packet go out of the cluster, get routed back and then\ntranslated to one of the backends.\n\nGiven that iptable rules are applied at the node boundary before any traffic from pods within\nthat node leave the node, the same arguments above apply here for replacing the cluster CIDR\nwith a representation of pod's nodeCIDR or it's interfaces.\n\n### iptables - accepting traffic after first packet, after being accepted by kubernetes rules\n\nThe [rule here currently](https://github.com/kubernetes/kubernetes/blob/v1.17.0/pkg/proxy/iptables/proxier.go#L1468-L1490)\nlooks as follows\n\n```go\n// The following rules can only be set if clusterCIDR has been defined.\nif len(proxier.clusterCIDR) != 0 {\n  // The following two rules ensure the traffic after the initial packet\n  // accepted by the \"kubernetes forwarding rules\" rule above will be\n  // accepted, to be as specific as possible the traffic must be sourced\n  // or destined to the clusterCIDR (to/from a pod).\n  writeLine(proxier.filterRules,\n    \"-A\", string(kubeForwardChain),\n    \"-s\", proxier.clusterCIDR,\n    \"-m\", \"comment\", \"--comment\", `\"kubernetes forwarding conntrack pod source rule\"`,\n    \"-m\", \"conntrack\",\n    \"--ctstate\", \"RELATED,ESTABLISHED\",\n    \"-j\", \"ACCEPT\",\n  )\n  writeLine(proxier.filterRules,\n    \"-A\", string(kubeForwardChain),\n    \"-m\", \"comment\", \"--comment\", `\"kubernetes forwarding conntrack pod destination rule\"`,\n    \"-d\", proxier.clusterCIDR,\n    \"-m\", \"conntrack\",\n    \"--ctstate\", \"RELATED,ESTABLISHED\",\n    \"-j\", \"ACCEPT\",\n  )\n}\n```\n\nThe interesting part of this rule that it already matches conntrack state to \"RELATED,ESTABLISHED\", \nwhich means that it does not apply to the initial packet, but after the connection has been setup and accepted.\n\nIn this case, dropping the `-d proxier.clusterCIDR` rule should have minimal impact on it behavior.\nWe would just be saying that if any connection is already established or related, just accept it.\n\nIn addition, since this rule is written after the rule to drop packets marked by `KUBE-MARK-DROP`,\nby the time we reach this rule, packets marked to dropped by kubernetes would already have been dropped.\nSo it should not break any kubernetes specific logic.\n\nUnfortunately in this case, it's not possible replace the cluster CIDR rule with local CIDR as\nthe traffic could be getting forwarded through this node to another node.\n\n### ipvs - masquerade off cluster traffic to services by node IP\n\nThe [rule here currently](https://github.com/kubernetes/kubernetes/blob/master/pkg/proxy/ipvs/proxier.go#L1649-L1654)\nlooks as follows.\n\n```go\n// This masquerades off-cluster traffic to a service VIP.  The idea\n// is that you can establish a static route for your Service range,\n// routing to any node, and that node will bridge into the Service\n// for you.  Since that might bounce off-node, we masquerade here.\n// If/when we support \"Local\" policy for VIPs, we should update this.\nwriteLine(proxier.natRules, append(args, \"dst,dst\", \"! -s\", proxier.clusterCIDR, \"-j\", string(KubeMarkMasqChain))...)\n```\n\nBy the same logic used in the first case for iptables, we can replace references to clusterCIDR with equivalent\nnode specific pod identification (node.podCIDR, interface prefix or bridge) to determine whether the traffic originated\nfrom within the cluster or not.\n\n### ipvs - accepting traffic after first packet, after being accepted by kubernetes rules\n\nThe [rule here currently](https://github.com/kubernetes/kubernetes/blob/master/pkg/proxy/ipvs/proxier.go#L1726-L1745)\nlooks as follows\n\n```go\n// The following two rules ensure the traffic after the initial packet\n// accepted by the \"kubernetes forwarding rules\" rule above will be\n// accepted, to be as specific as possible the traffic must be sourced\n// or destined to the clusterCIDR (to/from a pod).\nwriteLine(proxier.filterRules,\n\t\"-A\", string(KubeForwardChain),\n\t\"-s\", proxier.clusterCIDR,\n\t\"-m\", \"comment\", \"--comment\", `\"kubernetes forwarding conntrack pod source rule\"`,\n\t\"-m\", \"conntrack\",\n\t\"--ctstate\", \"RELATED,ESTABLISHED\",\n\t\"-j\", \"ACCEPT\",\n)\nwriteLine(proxier.filterRules,\n\t\"-A\", string(KubeForwardChain),\n\t\"-m\", \"comment\", \"--comment\", `\"kubernetes forwarding conntrack pod destination rule\"`,\n\t\"-d\", proxier.clusterCIDR,\n\t\"-m\", \"conntrack\",\n\t\"--ctstate\", \"RELATED,ESTABLISHED\",\n\t\"-j\", \"ACCEPT\",\n)\n```\nAgain, applying similar logic to the last rule for iptables, the proposal here is to simplify this drop reference to\nthe proxy.clusterCIDR and just match on the connection state.\n\n### Risks and Mitigations\n\nThe biggest risk we have is that we are expanding the scope of the last rule to potentially include non-kubernetes\ntraffic. This is considered mostly safe as it does not break any of the intended drop behavior. Plus once the initial\nconnection has been accepted, assuming nodes are used for kubernetes workloads, it's highly unlikely that we would\nneed to not accept it later.\n\n## Design Details\n\nThe idea of ‘determine cluster originated traffic’ would be captured in a new go interface type within kube-proxy,\nwith different implementations of the interface. The kube-proxy implementation itself would just call method on\ninterface to get the match criteria to write in the rule.\n\nThe new behavior can be opted-in using two flags. The first to determine the mode to use for detection, and the\nother (optionally) being the value to use in that mode. This separation of mode and value has the nice property\nthat if we default the mode to \"cluster-cidr\", then the current `--cluster-cidr` flag can be used as is to get\nthe current behaviour. So upgrades with no changes retain current behavior.\n\n```\n--detect-local={cluster-cidr | node-cidr | pod-interface-prefix  | bridge}\n\n  the mode to use for detection local traffic. The default is cluster-cidr (current behavior)\n\n--cluster-cidr=\"cidr[,cidr,..]\"\n\n  the current --cluster-cidr flag. It will be enhanced to read a comma separated list of CIDRs so that more\n  than one can be specified if necessary. kube-proxy considers traffic as local if source is one\n  of the CIDR values. This is only used if `--detect-local=cluster-cidr` .\n\n--node-cidr[=\"cidr[,cidr,..]\"]\n\n  (optional) list of node CIDRs as a comma separated list. kube-proxy considers traffic as local if source is one\n  of the CIDR values. If value is not specified, or flag is omitted,  defaults to node.podCIDR property on the node.\n  This is only used if `--detect-local=node-cidr` .\n\n--pod-interface-prefix=\"prefix[,prefix,..]\"\n\n  kube-proxy considers traffic as local if originating from an interface which matches one of given\n  prefixes. string argument is a comma separated list of interface prefix names, without the ending '+'.\n  This is only used if `--detect-local=pod-interface-prefix` or `--detect-local=bridge`. In the case of\n  latter, the prefix is used as option to `--physdev-in name` match instead of just `--physdev-in` in\n  iptables.\n```\n\nGiven that we are handling a list of rules, the jump to `KUBE-MARK-MARQ` will be implemented with a\njump to a new chain `KUBE-MASQ-IF-NOT-LOCAL` which will then either return or jump to `KUBE-MARK-MASQ`\nas appriate. For example:\n\n```\n-A WHEREVER -blah -blah -blah -j MARK-MASQ-IF-NOT-LOCAL\n\n-A MARK-MASQ-IF-NOT-LOCAL -s 10.0.1.0/24 -j RETURN\n-A MARK-MASQ-IF-NOT-LOCAL -s 10.0.3.0/24 -j RETURN\n-A MARK-MASQ-IF-NOT-LOCAL -s 10.0.5.0/24 -j RETURN\n-A MARK-MASQ-IF-NOT-LOCAL -j KUBE-MARK-MASQ\n```\n\nFuture changes to detection of local traffic (say using things like mark etc) can be done by adding more options\nto the `--detect-local` mode flag with any appropriate additional flags.\n\n### Graduation Criteria\n\nThese additional flags will go through alpha, beta etc graduation as for any feature.\n\n## Implementation History\n\n2019-11-04 - Creation of the KEP\n2019-11-27 - Revision with Implementation Details\n\n## Drawbacks [optional]\n\nThe main caveat in this KEP is the relaxation of the accept rule for \"ESTABLISHED,RELATED\" packets. The other two rules\nhave equivalent implementations, as long as we continue to guarantee that pod traffic is routed at the node boundary\non _every_ and _all_ nodes that makes up the kubernetes cluster. This would not work if that assumption were to change.\n\n## Alternatives [optional]\n\n### Multiple cluster CIDR rules\nOne alternative to consider is to explicitly track a list of cluster CIDRs in the ip table rules. If we \nwant to do this, we might want to consider making the cluster CIDR a first class resource, which we want to avoid.\n\nInstead in most cases, where the interface prefix is mostly fixed or we are using the `node.spec.podCIDR` attribute,\nchanges to the cluster CIDR does not need any change to the kube-proxy arguments or a restart, which we believe \nis of benefit when managing clusters.\n\n### ip-masq-agent like behavior\nThe other alternative is to have kube-proxy never track it and instead use something like \n[ip-masq-agent](https://kubernetes.io/docs/tasks/administer-cluster/ip-masq-agent/) to track what we masquerade\nor not. In this case, it assumes more knowledge from the users, but it does provide for a single place to update\nthese cidrs using existing tooling.\n"
+  },
+  {
+    "id": "c4df2966b1cfd0ca0539aaf6d79e16cc",
+    "title": "Adding AppProtocol to Services and Endpoints",
+    "authors": ["@robscott"],
+    "owningSig": "sig-network",
+    "participatingSigs": null,
+    "reviewers": ["@thockin", "@dcbw"],
+    "approvers": ["@thockin", "@dcbw"],
+    "editor": "",
+    "creationDate": "2019-12-27",
+    "lastUpdated": "2019-12-27",
+    "status": "implementable",
+    "seeAlso": [
+      "/keps/sig-network/20190603-EndpointSlice-API.md",
+      "https://github.com/kubernetes/kubernetes/issues/40244"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Adding AppProtocol to Services and Endpoints\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n- [Proposal](#proposal)\n    - [Services:](#services)\n    - [Endpoints:](#endpoints)\n  - [Risks and Mitigations](#risks-and-mitigations)\n  - [Graduation Criteria](#graduation-criteria)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nKubernetes does not have a standardized way of representing application\nprotocols. When a protocol is specified, it must be one of TCP, UDP, or SCTP.\nWith the EndpointSlice beta release in 1.17, a concept of AppProtocol was added\nthat would allow application protocols to be specified for each port. This KEP\nproposes adding support for that same attribute to Services and Endpoints.\n\n## Motivation\n\nThe lack of direct support for specifying application protocols for ports has\nled to widespread use of annotations, providing a poor user experience and\ngeneral frustration (https://github.com/kubernetes/kubernetes/issues/40244). \nUnfortunately annotations are cloud specific and simply can't provide the ease\nof use of a built in attribute like `AppProtocol`. Since application protocols\nare specific to each port specified on a Service or Endpoints resource, it makes\nsense to have a way to specify it at that level.\n\n### Goals\n\nAdd AppProtocol field to Ports in Services and Endpoints.\n\n## Proposal\n\nIn both Endpoints and Services, a new `AppProtocol` field would be added. In\nboth cases, constraints validation would directly mirror what already exists\nwith EndpointSlices.\n\n#### Services:\n```go\n// ServicePort represents the port on which the service is exposed\ntype ServicePort struct {\n    ...\n    // The application protocol for this port.\n    // This field follows standard Kubernetes label syntax.\n    // Un-prefixed names are reserved for IANA standard service names (as per\n    // RFC-6335 and http://www.iana.org/assignments/service-names).\n    // Non-standard protocols should use prefixed names such as\n    // mycompany.com/my-custom-protocol.\n    // +optional\n    AppProtocol *string\n}\n```\n\n#### Endpoints:\n```go\n// EndpointPort is a tuple that describes a single port.\ntype EndpointPort struct {\n    ...\n    // The application protocol for this port.\n    // This field follows standard Kubernetes label syntax.\n    // Un-prefixed names are reserved for IANA standard service names (as per\n    // RFC-6335 and http://www.iana.org/assignments/service-names).\n    // Non-standard protocols should use prefixed names such as\n    // mycompany.com/my-custom-protocol.\n    // +optional\n    AppProtocol *string\n}\n```\n\n### Risks and Mitigations\n\nIt may take some time for cloud providers and other consumers of these APIs to\nsupport this attribute. To help with this, we will work to communicate this\nchange well in advance of release so it can be well supported initially.\n\n### Graduation Criteria\n\nThis adds a new optional attribute to 2 existing stable APIs. There is no need\nfor feature gating or a graduation process, this will be added to the existing\nAPI versions.\n"
+  },
+  {
+    "id": "9b51e3288b2042f10a86521ab732844e",
+    "title": "Protomote sysctl annotations to fields",
+    "authors": ["@ingvagabund"],
+    "owningSig": "sig-node",
+    "participatingSigs": ["sig-auth"],
+    "reviewers": ["@sjenning", "@derekwaynecarr"],
+    "approvers": ["@sjenning ", "@derekwaynecarr"],
+    "editor": "",
+    "creationDate": "2018-04-30",
+    "lastUpdated": "2018-05-02",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Promote sysctl annotations to fields\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Promote annotations to fields](#promote-annotations-to-fields)\n  - [Promote \u003ccode\u003e--experimental-allowed-unsafe-sysctls\u003c/code\u003e kubelet flag to kubelet config api option](#promote--kubelet-flag-to-kubelet-config-api-option)\n  - [Gate the feature](#gate-the-feature)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nSetting the `sysctl` parameters through annotations provided a successful story\nfor defining better constraints of running applications.\nThe `sysctl` feature has been tested by a number of people without any serious\ncomplaints. Promoting the annotations to fields (i.e. to beta) is another step in making the\n`sysctl` feature closer towards the stable API.\n\nCurrently, the `sysctl` provides `security.alpha.kubernetes.io/sysctls` and `security.alpha.kubernetes.io/unsafe-sysctls` annotations that can be used\nin the following way:\n  ```yaml\n  apiVersion: v1\n  kind: Pod\n  metadata:\n    name: sysctl-example\n    annotations:\n      security.alpha.kubernetes.io/sysctls: kernel.shm_rmid_forced=1\n      security.alpha.kubernetes.io/unsafe-sysctls: net.ipv4.route.min_pmtu=1000,kernel.msgmax=1 2 3\n  spec:\n    ...\n  ```\n\n  The goal is to transition into native fields on pods:\n\n  ```yaml\n  apiVersion: v1\n  kind: Pod\n  metadata:\n    name: sysctl-example\n  spec:\n    securityContext:\n      sysctls:\n      - name: kernel.shm_rmid_forced\n        value: 1\n      - name: net.ipv4.route.min_pmtu\n        value: 1000\n        unsafe: true\n      - name: kernel.msgmax\n        value: \"1 2 3\"\n        unsafe: true\n    ...\n  ```\n\nThe `sysctl` design document with more details and rationals is available at [design-proposals/node/sysctl.md](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/node/sysctl.md#pod-api-changes)\n\n## Motivation\n\nAs mentioned in [contributors/devel/api_changes.md#alpha-field-in-existing-api-version](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api_changes.md#alpha-field-in-existing-api-version):\n\n\u003e Previously, annotations were used for experimental alpha features, but are no longer recommended for several reasons:\n\u003e\n\u003e    They expose the cluster to \"time-bomb\" data added as unstructured annotations against an earlier API server (https://issue.k8s.io/30819)\n\u003e    They cannot be migrated to first-class fields in the same API version (see the issues with representing a single value in multiple places in backward compatibility gotchas)\n\u003e\n\u003e The preferred approach adds an alpha field to the existing object, and ensures it is disabled by default:\n\u003e\n\u003e ...\n\nThe annotations as a means to set `sysctl` are no longer necessary.\nThe original intent of annotations was to provide additional description of Kubernetes\nobjects through metadata.\nIt's time to separate the ability to annotate from the ability to change sysctls settings\nso a cluster operator can elevate the distinction between experimental and supported usage\nof the feature.\n\n### Promote annotations to fields\n\n* Introduce native `sysctl` fields in pods through `spec.securityContext.sysctl` field as:\n\n  ```yaml\n  sysctl:\n  - name: SYSCTL_PATH_NAME\n    value: SYSCTL_PATH_VALUE\n    unsafe: true    # optional field\n  ```\n\n* Introduce native `sysctl` fields in [PSP](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) as:\n\n  ```yaml\n  apiVersion: v1\n  kind: PodSecurityPolicy\n  metadata:\n    name: psp-example\n  spec:\n    sysctls:\n    - kernel.shmmax\n    - kernel.shmall\n    - net.*\n  ```\n\n  More examples at [design-proposals/node/sysctl.md#allowing-only-certain-sysctls](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/node/sysctl.md#allowing-only-certain-sysctls)\n\n### Promote `--experimental-allowed-unsafe-sysctls` kubelet flag to kubelet config api option\n\nAs there is no longer a need to consider the `sysctl` feature experimental,\nthe list of unsafe sysctls can be configured accordingly through:\n\n```go\n// KubeletConfiguration contains the configuration for the Kubelet\ntype KubeletConfiguration struct {\n  ...\n  // Whitelist of unsafe sysctls or unsafe sysctl patterns (ending in *).\n  // Default: nil\n  // +optional\n  AllowedUnsafeSysctls []string `json:\"allowedUnsafeSysctls,omitempty\"`\n}\n```\n\nUpstream issue: https://github.com/kubernetes/kubernetes/issues/61669\n\n### Gate the feature\n\nAs the `sysctl` feature stabilizes, it's time to gate the feature [1] and enable it by default.\n\n* Expected feature gate key: `Sysctls`\n* Expected default value: `true`\n\nWith the `Sysctl` feature enabled, both sysctl fields in `Pod` and `PodSecurityPolicy`\nand the whitelist of unsafed sysctls are acknowledged.\nIf disabled, the fields and the whitelist are just ignored.\n\n[1] https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/\n\n## Proposal\n\nThis is where we get down to the nitty gritty of what the proposal actually is.\n\n### User Stories\n\n* As a cluster admin, I want to have `sysctl` feature versioned so I can assure backward compatibility\n  and proper transformation between versioned to internal representation and back..\n* As a cluster admin, I want to be confident the `sysctl` feature is stable enough and well supported so\n  applications are properly isolated\n* As a cluster admin, I want to be able to apply the `sysctl` constraints on the cluster level so\n  I can define the default constraints for all pods.\n\n### Implementation Details/Notes/Constraints\n\nExtending `SecurityContext` struct with `Sysctls` field:\n\n```go\n// PodSecurityContext holds pod-level security attributes and common container settings.\n// Some fields are also present in container.securityContext.  Field values of\n// container.securityContext take precedence over field values of PodSecurityContext.\ntype PodSecurityContext struct {\n    ...\n    // Sysctls is a white list of allowed sysctls in a pod spec.\n    Sysctls []Sysctl `json:\"sysctls,omitempty\"`\n}\n```\n\nExtending `PodSecurityPolicySpec` struct with `Sysctls` field:\n\n```go\n// PodSecurityPolicySpec defines the policy enforced on sysctls.\ntype PodSecurityPolicySpec struct {\n    ...\n    // Sysctls is a white list of allowed sysctls in a pod spec.\n    Sysctls []Sysctl `json:\"sysctls,omitempty\"`\n}\n```\n\nFollowing steps in [devel/api_changes.md#alpha-field-in-existing-api-version](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api_changes.md#alpha-field-in-existing-api-version)\nduring implementation.\n\nValidation checks implemented as part of [#27180](https://github.com/kubernetes/kubernetes/pull/27180).\n\n### Risks and Mitigations\n\nWe need to assure backward compatibility, i.e. object specifications with `sysctl` annotations\nmust still work after the graduation.\n\n## Graduation Criteria\n\n* API changes allowing to configure the pod-scoped `sysctl` via `spec.securityContext` field.\n* API changes allowing to configure the cluster-scoped `sysctl` via `PodSecurityPolicy` object\n* Promote `--experimental-allowed-unsafe-sysctls` kubelet flag to kubelet config api option\n* feature gate enabled by default\n* e2e tests\n\n## Implementation History\n\nThe `sysctl` feature is tracked as part of [features#34](https://github.com/kubernetes/features/issues/34).\nThis is one of the goals to promote the annotations to fields.\n"
+  },
+  {
+    "id": "defcb62956b7951aecf33b62ae7778a9",
+    "title": "Efficient Node Heartbeat",
+    "authors": ["@wojtek-t", "with input from @bgrant0607, @dchen1107, @yujuhong, @lavalamp"],
+    "owningSig": "sig-node",
+    "participatingSigs": ["sig-scalability", "sig-api-machinery", "sig-scheduling"],
+    "reviewers": ["@deads2k", "@lavalamp"],
+    "approvers": ["@dchen1107", "@derekwaynecarr"],
+    "editor": "TBD",
+    "creationDate": "2018-04-27",
+    "lastUpdated": "2018-04-27",
+    "status": "implemented",
+    "seeAlso": [
+      "https://github.com/kubernetes/kubernetes/issues/14733",
+      "https://github.com/kubernetes/kubernetes/pull/14735"
+    ],
+    "replaces": ["n/a"],
+    "supersededBy": ["n/a"],
+    "markdown": "\n# Efficient Node Heartbeats\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Risks and Mitigations](#risks-and-mitigations)\n  - [Testing Plan](#testing-plan)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n  - [Dedicated “heartbeat” object instead of “leader election” one](#dedicated-heartbeat-object-instead-of-leader-election-one)\n  - [Events instead of dedicated heartbeat object](#events-instead-of-dedicated-heartbeat-object)\n  - [Reuse the Component Registration mechanisms](#reuse-the-component-registration-mechanisms)\n  - [Split Node object into two parts at etcd level](#split-node-object-into-two-parts-at-etcd-level)\n  - [Delta compression in etcd](#delta-compression-in-etcd)\n  - [Replace etcd with other database](#replace-etcd-with-other-database)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nNode heartbeats are necessary for correct functioning of Kubernetes cluster.\nThis proposal makes them significantly cheaper from both scalability and\nperformance perspective.\n\n## Motivation\n\nWhile running different scalability tests we observed that in big enough clusters\n(more than 2000 nodes) with non-trivial number of images used by pods on all\nnodes (10-15), we were hitting etcd limits for its database size. That effectively\nmeans that etcd enters \"alert mode\" and stops accepting all write requests.\n\nThe underlying root cause is combination of:\n\n- etcd keeping both current state and transaction log with copy-on-write\n- node heartbeats being pontetially very large objects (note that images\n  are only one potential problem, the second are volumes and customers\n  want to mount 100+ volumes to a single node) - they may easily exceed 15kB;\n  even though the patch send over network is small, in etcd we store the\n\twhole Node object\n- Kubelet sending heartbeats every 10s\n\nThis proposal presents a proper solution for that problem.\n\n\nNote that currently (by default):\n\n- Lack of NodeStatus update for `\u003cnode-monitor-grace-period\u003e` (default: 40s)\n  results in NodeController marking node as NotReady (pods are no longer\n  scheduled on that node)\n- Lack of NodeStatus updates for `\u003cpod-eviction-timeout\u003e` (default: 5m)\n  results in NodeController starting pod evictions from that node\n\nWe would like to preserve that behavior.\n\n\n### Goals\n\n- Reduce size of etcd by making node heartbeats cheaper\n\n### Non-Goals\n\nThe following are nice-to-haves, but not primary goals:\n\n- Reduce resource usage (cpu/memory) of control plane (e.g. due to processing\n  less and/or smaller objects)\n- Reduce watch-related load on Node objects\n\n## Proposal\n\nWe propose introducing a new `Lease` built-in API in the newly create API group\n`coordination.k8s.io`. To make it easily reusable for other purposes it will\nbe namespaced. Its schema will be as following:\n\n```\ntype Lease struct {\n  metav1.TypeMeta `json:\",inline\"`\n  // Standard object's metadata.\n  // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata\n  // +optional\n  ObjectMeta metav1.ObjectMeta `json:\"metadata,omitempty\"`\n\n  // Specification of the Lease.\n  // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status\n  // +optional\n  Spec LeaseSpec `json:\"spec,omitempty\"`\n}\n\ntype LeaseSpec struct {\n  HolderIdentity       string           `json:\"holderIdentity\"`\n  LeaseDurationSeconds int32            `json:\"leaseDurationSeconds\"`\n  AcquireTime          metav1.MicroTime `json:\"acquireTime\"`\n  RenewTime            metav1.MicroTime `json:\"renewTime\"`\n  LeaseTransitions     int32            `json:\"leaseTransitions\"`\n}\n```\n\nThe Spec is effectively of already existing (and thus proved) [LeaderElectionRecord][].\nThe only difference is using `MicroTime` instead of `Time` for better precision.\nThat would hopefully allow us go get directly to Beta.\n\nWe will use that object to represent node heartbeat - for each Node there will\nbe a corresponding `Lease` object with Name equal to Node name in a newly\ncreated dedicated namespace (we considered using `kube-system` namespace but\ndecided that it's already too overloaded).\nThat namespace should be created automatically (similarly to \"default\" and\n\"kube-system\", probably by NodeController) and never be deleted (so that nodes\ndon't require permission for it).\n\nWe considered using CRD instead of built-in API. However, even though CRDs are\n`the new way` for creating new APIs, they don't yet have versioning support\nand are significantly less performant (due to lack of protobuf support yet).\nWe also don't know whether we could seamlessly transition storage from a CRD\nto a built-in API if we ran into a performance or any other problems.\nAs a result, we decided to proceed with built-in API.\n\n\nWith this new API in place, we will change Kubelet so that:\n\n1. Kubelet is periodically computing NodeStatus every 10s (at it is now), but that will\n   be independent from reporting status\n1. Kubelet is reporting NodeStatus if:\n   - there was a meaningful change in it (initially we can probably assume that every\n     change is meaningful, including e.g. images on the node)\n   - or it didn’t report it over last `node-status-update-period` seconds\n1. Kubelet creates and periodically updates its own Lease object and frequency\n   of those updates is independent from NodeStatus update frequency.\n\nIn the meantime, we will change `NodeController` to treat both updates of NodeStatus\nobject as well as updates of the new `Lease` object corresponding to a given\nnode as healthiness signal from a given Kubelet. This will make it work for both old\nand new Kubelets.\n\nWe should also:\n\n1. audit all other existing core controllers to verify if they also don’t require\n   similar changes in their logic ([ttl controller][] being one of the examples)\n1. change controller manager to auto-register that `Lease` CRD\n1. ensure that `Lease` resource is deleted when corresponding node is\n   deleted (probably via owner references)\n1. [out-of-scope] migrate all LeaderElection code to use that CRD\n\nOnce all the code changes are done, we will:\n\n1. start updating `Lease` object every 10s by default, at the same time\n   reducing frequency of NodeStatus updates initially to 40s by default.\n   We will reduce it further later.\n   Note that it doesn't reduce frequency by which Kubelet sends \"meaningful\"\n   changes - it only impacts the frequency of \"lastHeartbeatTime\" changes.\n1. announce that we are going to reduce frequency of NodeStatus updates further\n   and give people 1-2 releases to switch their code to use `Lease`\n   object (if they relied on frequent NodeStatus changes)\n1. further reduce NodeStatus updates frequency to not less often than once per\n   1 minute.\n   We can’t stop periodically updating NodeStatus as it would be API breaking change,\n   but it’s fine to reduce its frequency (though we should continue writing it at\n   least once per eviction period).\n\nBased on experiments, unsuccessful attempt to enable `Server Side Apply` due to\nperformance issues it was causing, in v1.17 release, we will:\n- reduce frequency of NodeStatus update to once every 5 minutes instead of\n  1 minute (note that if any condition changes, the update will happen within\n  next 10 seconds anyway, so we are only talking about periodic heartbeats)\n- reduce frequency of updating NodeStatuses in NodeProblemDetector also to\n  5 minutes (to keep those two frequencies in sync) (similarly as Kubelet,\n  if any condition changes, update will happen within next 30s as it is now)\n\nObviously, in both cases, we are only talking about default values, which may\nbe changed by cluster operators if needed.\nWe don't want to increase it further, as we would like to keep at least one\nNodeStatus update at least every eviction period (which is defaulted to 5 minutes).\n\nThis change will help with enabling server-side-apply for all operation and will\ngive more performance slack in largest supported clusters. As an example, assuming\naverage node object size 20KB (which happens in production clusters), that will\njump to ~30KB with server-side-apply feature, in 5k-node clusters, we are reducing\netcd write throughput from 150MB/min to 30MB/min. Note that this isn't the only\ngain, because we also save on sending watch events, in kube-apiserver, and so on.\n\nAdditionally, we will modify kubectl describe to show the information from the\ncorresponding Lease object.\n\nWith those two changes and positive user-feedback we already got, we will also\ngraduate the feature to GA in 1.17 too.\n\nOther notes:\n\n1. Additional advantage of using Lease for that purpose would be the\n   ability to exclude it from audit profile and thus reduce the audit logs footprint.\n\n[LeaderElectionRecord]: https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/client-go/tools/leaderelection/resourcelock/interface.go#L37\n[ttl controller]: https://github.com/kubernetes/kubernetes/blob/master/pkg/controller/ttl/ttl_controller.go#L155\n[NPD documentation]: https://kubernetes.io/docs/tasks/debug-application-cluster/monitor-node-health/\n[kubernetes/kubernetes#63667]: https://github.com/kubernetes/kubernetes/issues/63677\n\n### Risks and Mitigations\n\nIncreasing default frequency of NodeStatus updates may potentially break clients\nrelying on frequent Node object updates. However, in non-managed solutions, customers\nwill still be able to restore previous behavior by setting appropriate flag values.\nThus, changing defaults to what we recommend is the path to go with.\n\nTo evaluate the risk of further reduction of NodeStatus updates, I went through\ncouple thousands of files mentioning `LastHeartbeatTime` and `k8s` on github to\nunderstand the potential of breaking some system. There are couple places that\nare incompatible with node leases, but they would already be broken by current\n1m frequency, e.g.:\n- [mesos status updater](https://github.com/kubernetes-retired/kube-mesos-framework/blob/9bee6ea903eb7c3e78b9b0a4accb9b8162cabeb5/pkg/node/statusupdater.go#L122)\n- [node cacher](https://github.com/sandflee/k8s-load-simulator/blob/6d9a1c7458df1231e06d6eb7ac9eecb4f7d0e3ce/pkg/node/node_cacher.go#L25)\n- [isk8salive](https://github.com/barkbay/isk8salive/blob/a47c0aa46c1008fab4042c4047e9c5ee68516845/main.go#L55)\n- [node evictor](https://github.com/jdartigalongue/k8s-node-evictor/blob/fd03ee3452951bb191dde792eada7a29549594cd/controller.go#L88)\n- [node cleanup](https://github.com/monder/aws-node-cleanup/blob/25b2a568fa303de053b11e8387f5ddbc8c192ea6/node-cleanup.go#L21)\n\nThat said, it sounds better to update the frequency before going to GA.\n\n### Testing Plan\n\nThere is a set of dedicated end-to-end tests added for that feature excercising:\n- whether Lease object is being created and update by Kubelet\n  (gce-cos-master-default)\n- whether Kubelet is reducing frequency of node status updates appropriately\n  (gce-cos-master-default)\n- whether Lease object is deleted on node deletion (gce-cos-master-serial)\n\nAdditionally, if the feature gate is switched on, all existing test suites are\nimplicitly testing behavior of this feature, as this is then the signal for\nhealthiness of nodes.\n\nAdditionally, the main benefit from this feature is obviously performance and\nscalability. For this purpose, as part of all scalability tests, we are\nadditionally measuring maximum etcd database size.\n\n\n## Graduation Criteria\n\nThe API can be immediately promoted to Beta, as the API is effectively a copy of\nalready existing LeaderElectionRecord. It will be promoted to GA once it's gone\na sufficient amount of time as Beta with no changes.\n\nThe changes in components logic (Kubelet, NodeController) should be done behind\na feature gate. We suggest making that enabled by default once the feature is\nimplemented.\n\nBeta:\n- Confirmed scalability/performance gain: decreased of total etcd size by 2x+ on\n5k-node clusters and no drop in any other scalability SLIs (in fact we observed\ndecrease in API call latencies by up to 20-30% for some resources). Verified on\nboth real clusters and Kubemark.\n\nGA:\n- Enabled by default for a release with no complaints.\n- Frequency of NodeStatus defaulted to 5m\n- kubectl describe node showing information from corresponding Lease object\n\n\n## Implementation History\n\n- v1.11: KEP Summary, Motivation and Proposal merged\n- v1.13: Feature launched to Alpha (default: off)\n- v1.14: Feature launched to Beta (default: on)\n- v1.16: Minor improvements based on user feedback (e.g. [80429][], [81174][])\n\n[80429]: https://github.com/kubernetes/kubernetes/pull/80429\n[81174]: https://github.com/kubernetes/kubernetes/pull/81174\n\n## Alternatives\n\nWe considered a number of alternatives, most important mentioned below.\n\n### Dedicated “heartbeat” object instead of “leader election” one\n\nInstead of introducing and using “lease” object, we considered\nintroducing a dedicated “heartbeat” object for that purpose. Apart from that,\nall the details about the solution remain pretty much the same.\n\nPros:\n\n- Conceptually easier to understand what the object is for\n\nCons:\n\n- Introduces a new, narrow-purpose API. Lease is already used by other\n  components, implemented using annotations on Endpoints and ConfigMaps.\n\n### Events instead of dedicated heartbeat object\n\nInstead of introducing a dedicated object, we considered using “Event” object\nfor that purpose. At the high-level the solution looks very similar. \nThe differences from the initial proposal are:\n\n- we use existing “Event” api instead of introducing a new API\n- we create a dedicated namespace; events that should be treated as healthiness\n  signal by NodeController will be written by Kubelets (unconditionally) to that\n  namespace\n- NodeController will be watching only Events from that namespace to avoid\n  processing all events in the system (the volume of all events will be huge)\n- dedicated namespace also helps with security - we can give access to write to\n  that namespace only to Kubelets\n\nPros:\n\n- No need to introduce new API\n   - We can use that approach much earlier due to that.\n- We already need to optimize event throughput - separate etcd instance we have\n  for them may help with tuning\n- Low-risk roll-forward/roll-back: no new objects is involved (node controller\n  starts watching events, kubelet just reduces the frequency of heartbeats)\n\nCons:\n\n- Events are conceptually “best-effort” in the system:\n   - they may be silently dropped in case of problems in the system (the event recorder\n     library doesn’t retry on errors, e.g. to not make things worse when control-plane\n     is starved)\n   - currently, components reporting events don’t even know if it succeeded or not (the\n     library is built in a way that you throw the event into it and are not notified if\n     that was successfully submitted or not).\n     Kubelet sending any other update has full control on how/if retry errors.\n   - lack of fairness mechanisms means that even when some events are being successfully\n     send, there is no guarantee that any event from  a given Kubelet will be submitted\n     over a given time period\n\tSo this would require a different mechanism of reporting those “heartbeat” events.\n- Once we have “request priority” concept, I think events should have the lowest one.\n  Even though no particular heartbeat is important, guarantee that some heartbeats will\n  be successfully send it crucial (not delivering any of them will result in unnecessary\n  evictions or not-scheduling to a given node). So heartbeats should be of the highest\n  priority. OTOH, node heartbeats are one of the most important things in the system\n  (not delivering them may result in unnecessary evictions), so they should have the\n  highest priority.\n- No core component in the system is currently watching events\n   - it would make system’s operation harder to explain\n- Users watch Node objects for heartbeats (even though we didn’t recommend it).\n  Introducing a new object for the purpose of heartbeat will allow those users to\n  migrate, while using events for that purpose breaks that ability. (Watching events\n  may put us in tough situation also from performance reasons.)\n- Deleting all events (e.g. event etcd failure + playbook response) should continue to\n  not cause a catastrophic failure and the design will need to account for this.\n\n### Reuse the Component Registration mechanisms\n\nKubelet is one of control-place components (shared controller). Some time ago, Component\nRegistration proposal converged into three parts:\n\n- Introducing an API for registering non-pod endpoints, including readiness information: #18610\n- Changing endpoints controller to also watch those endpoints\n- Identifying some of those endpoints as “components”\n\nWe could reuse that mechanism to represent Kubelets as non-pod endpoint API.\n\nPros:\n\n- Utilizes desired API\n\nCons:\n\n- Requires introducing that new API\n- Stabilizing the API would take some time\n- Implementing that API requires multiple changes in different components\n\n### Split Node object into two parts at etcd level\n\nWe may stick to existing Node API and solve the problem at storage layer. At the\nhigh level, this means splitting the Node object into two parts in etcd (frequently\nmodified one and the rest).\n\nPros:\n\n- No need to introduce new API\n- No need to change any components other than kube-apiserver\n\nCons:\n\n- Very complicated to support watch\n- Not very generic (e.g. splitting Spec and Status doesn’t help, it needs to be just\n  heartbeat part)\n- [minor] Doesn’t reduce amount of data that should be processed in the system (writes,\n  reads, watches, …)\n\n### Delta compression in etcd\n\nAn alternative for the above can be solving this completely at the etcd layer. To\nachieve that, instead of storing full updates in etcd transaction log, we will just\nstore “deltas” and snapshot the whole object only every X seconds/minutes.\n\nPros:\n\n- Doesn’t require any changes to any Kubernetes components\n\nCons:\n\n- Computing delta is tricky (etcd doesn’t understand Kubernetes data model, and\n  delta between two protobuf-encoded objects is not necessary small)\n- May require a major rewrite of etcd code and not even be accepted by its maintainers\n- More expensive computationally to get an object in a given resource version (which\n  is what e.g. watch is doing)\n\n### Replace etcd with other database\n\nInstead of using etcd, we may also consider using some other open-source solution.\n\nPros:\n\n- Doesn’t require new API\n\nCons:\n\n- We don’t even know if there exists solution that solves our problems and can be used.\n- Migration will take us years.\n"
+  },
+  {
+    "id": "a5c17f694b00b8bd737e3ad2d67d1eef",
+    "title": "Node Topology Manager",
+    "authors": ["@ConnorDoyle", "@balajismaniam", "@lmdaly"],
+    "owningSig": "sig-node",
+    "participatingSigs": ["sig-node"],
+    "reviewers": [
+      "@vikasc",
+      "@derekwaynecarr",
+      "@jeremyeder",
+      "@RenaudWasTaken",
+      "@klueska",
+      "@nolancon"
+    ],
+    "approvers": ["@dawnchen", "@derekwaynecarr"],
+    "editor": "Louise Daly",
+    "creationDate": "2019-01-30",
+    "lastUpdated": "2019-01-30",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Node Topology Manager\n\n_Authors:_\n\n* @ConnorDoyle - Connor Doyle \u0026lt;connor.p.doyle@intel.com\u0026gt;\n* @balajismaniam - Balaji Subramaniam \u0026lt;balaji.subramaniam@intel.com\u0026gt;\n* @lmdaly - Louise M. Daly \u0026lt;louise.m.daly@intel.com\u0026gt;\n\n_Reviewers:_\n* @klueska - Kevin Klues \u0026lt;kklues@nvidia.com\u0026gt;\n* @nolancon - Conor Nolan \u0026lt;conor.nolan@intel.com\u0026gt;\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Overview](#overview)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n  - [User Stories](#user-stories)\n- [Proposal](#proposal)\n  - [Proposed Changes](#proposed-changes)\n    - [New Component: Topology Manager](#new-component-topology-manager)\n      - [Policies](#policies)\n      - [Computing Preferred Affinity](#computing-preferred-affinity)\n      - [New Interfaces](#new-interfaces)\n    - [Feature Gate and Kubelet Flags](#feature-gate-and-kubelet-flags)\n    - [Changes to Existing Components](#changes-to-existing-components)\n- [Graduation Criteria](#graduation-criteria)\n  - [Phase 1: Alpha (v1.16) [COMPLETED]](#phase-1-alpha-v116-completed)\n  - [Phase 2: Beta (target v1.17)](#phase-2-beta-target-v117)\n  - [GA (stable)](#ga-stable)\n- [Challenges](#challenges)\n- [Limitations](#limitations)\n- [Alternatives](#alternatives)\n- [References](#references)\n\u003c!-- /toc --\u003e\n\n# Overview\n\nAn increasing number of systems leverage a combination of CPUs and\nhardware accelerators to support latency-critical execution and\nhigh-throughput parallel computation. These include workloads in fields\nsuch as telecommunications, scientific computing, machine learning,\nfinancial services and data analytics. Such hybrid systems comprise a\nhigh performance environment.\n\nIn order to extract the best performance, optimizations related to CPU\nisolation and memory and device locality are required. However, in\nKubernetes, these optimizations are handled by a disjoint set of\ncomponents.\n\nThis proposal provides a mechanism to coordinate fine-grained hardware\nresource assignments for different components in Kubernetes.\n\n# Motivation\n\nMultiple components in the Kubelet make decisions about system\ntopology-related assignments:\n\n- CPU manager\n  - The CPU manager makes decisions about the set of CPUs a container is\nallowed to run on. The only implemented policy as of v1.8 is the static\none, which does not change assignments for the lifetime of a container.\n- Device manager\n  - The device manager makes concrete device assignments to satisfy\ncontainer resource requirements. Generally devices are attached to one\nperipheral interconnect. If the device manager and the CPU manager are\nmisaligned, all communication between the CPU and the device can incur\nan additional hop over the processor interconnect fabric.\n- Container Network Interface (CNI)\n  - NICs including SR-IOV Virtual Functions have affinity to one socket,\nwith measurable performance ramifications.\n\n*Related Issues:*\n\n- [Hardware topology awareness at node level (including NUMA)][k8s-issue-49964]\n- [Discover nodes with NUMA architecture][nfd-issue-84]\n- [Support VF interrupt binding to specified CPU][sriov-issue-10]\n- [Proposal: CPU Affinity and NUMA Topology Awareness][proposal-affinity]\n\nNote that all of these concerns pertain only to multi-socket systems. Correct\nbehavior requires that the kernel receive accurate topology information from\nthe underlying hardware (typically via the SLIT table). See section 5.2.16\nand 5.2.17 of the\n[ACPI Specification](http://www.acpi.info/DOWNLOADS/ACPIspec50.pdf) for more\ninformation.\n\n## Goals\n\n- Arbitrate preferred NUMA Node affinity for containers based on input from\n  CPU Manager and Device Manager.\n- Provide an internal interface and pattern to integrate additional\n  topology-aware Kubelet components.\n\n## Non-Goals\n\n- _Inter-device connectivity:_ Decide device assignments based on direct\n  device interconnects. This issue can be separated from socket\n  locality. Inter-device topology can be considered entirely within the\n  scope of the Device Manager, after which it can emit possible\n  socket affinities. The policy to reach that decision can start simple\n  and iterate to include support for arbitrary inter-device graphs.\n- _HugePages:_ This proposal assumes that pre-allocated HugePages are\n  spread among the available memory nodes in the system. We further assume\n  the operating system provides best-effort local page allocation for\n  containers (as long as sufficient HugePages are free on the local memory\n  node.\n- _CNI:_ Changing the Container Networking Interface is out of scope for\n  this proposal. However, this design should be extensible enough to\n  accommodate network interface locality if the CNI adds support in the\n  future. This limitation is potentially mitigated by the possibility to\n  use the device plugin API as a stopgap solution for specialized\n  networking requirements.\n\n## User Stories\n\n*Story 1: Fast virtualized network functions*\n\nA user asks for a \"fast network\" and automatically gets all the various\npieces coordinated (hugepages, cpusets, network device) in a preferred NUMA Node\nalignment, in most cases this will be the narrrowest possible set of NUMA Node(s).\n\n*Story 2: Accelerated neural network training*\n\nA user asks for an accelerator device and some number of exclusive CPUs\nin order to get the best training performance, due to NUMA Node alignment of\nthe assigned CPUs and devices.\n\n# Proposal\n\n*Main idea: Two phase topology coherence protocol*\n\nTopology affinity is tracked at the container level, similar to devices and\nCPU affinity. At pod admission time, a new component called the Topology\nManager collects possible configurations from the Device Manager and the\nCPU Manager. The Topology Manager acts as an oracle for local alignment by\nthose same components when they make concrete resource allocations. We\nexpect the consulted components to use the inferred QoS class of each\npod in order to prioritize the importance of fulfilling optimal locality.\n\n## Proposed Changes\n\n### New Component: Topology Manager\n\nThis proposal is focused on a new component in the Kubelet called the\nTopology Manager. The Topology Manager implements the pod admit handler\ninterface and participates in Kubelet pod admission. When the `Admit()`\nfunction is called, the Topology Manager collects topology hints from other\nKubelet components.\n\nIf the hints are not compatible, the Topology Manager may choose to\nreject the pod. Behavior in this case depends on a new Kubelet configuration\nvalue to choose the topology policy. The Topology Manager supports four\npolicies: `none`(default), `best-effort`, `restricted` and `single-numa-node`. \n\nA topology hint indicates a preference for some well-known local resources.\nThe Topology Hint currently consists of \n* A list of bitmasks denoting the possible NUMA Nodes where a request can be satisfied.\n* A preferred field.\n    * This field is defined as follows:\n      * For each Hint Provider, there is a possible resource assignment that satisfies the request, such that the least possible number of NUMA nodes is involved (caculated as if the node were empty.)\n      * There is a possible assignment where the union of involved NUMA nodes for all such resource is no larger than the width required for any single resource.\n\n#### Policies\n\n**none (default)**: Kubelet does not consult Topology Manager for placement decisions. \n\n**best-effort**: Topology Manager will provide the preferred allocation for the container based\non the hints provided by the Hint Providers. If an undesirable allocation is determined, the pod will be admitted with this undesirable allocation.\n\n**restricted**: Topology Manager will provide the preferred allocation for the container based\non the hints provided by the Hint Providers. If an undesirable allocation is determined, the pod will be rejected. \nThis will result in the pod being in a `Terminated` state, with a pod admission failure.\n\n**single-numa-node**: Topology mananager will enforce an allocation of all resources on a single NUMA Node. If such\nan allocation is not possible, the pod will be rejected. This will result in the pod being in a `Terminated` state, with a pod admission failure.\n\nThe Topology Manager component will be disabled behind a feature gate until\ngraduation from alpha to beta.\n\n#### Computing Preferred Affinity\n\nAfter collecting hints from all providers, the Topology Manager preforms the\naffinity calcuation to determine the best fit Topology Hint.\n\nThe chosen Topology Manager policy then decicds to admit or reject the pod based on this hint.\n\n**Affinity Calcuation:**\n1. Loops through the list of hint providers and saves an accumulated list of \n   the hints returned by each hint provider.\n2. Iterates through all permutations of hints accumulated in Step 1. The hint affinites are merged to a single hint\n   by performing a bitwise AND. The preferred field on the merged hint is set to false if any of the hints in the \n   permutation returned a false preferred.\n3. The hint with the narrowest preferred affinity is returned.\n   * Narrowest in this case means the least number of NUMA nodes required to satisfy the resource request.      \n4. If no hint with at least one NUMA Node set is found, return a default hint which is a hint\n   with all NUMA Nodes set and preferred set to false.\n\n**Policy Decisions:**\n\n- **best-effort**\n    * Admits the pod to the node regardless of the Topology Hint stored.\n- **restricted**:\n    * Admits the pod to the node if the preferred field of the Topology Hint is set to true.\n- **single-numa-node**:\n    * Admits the pod to the node if the preferred field of the Topology is set to true **and** the bitmask is set to a single NUMA node.\n\n#### New Interfaces\n\n```go\npackage bitmask\n\n// BitMask interface allows hint providers to create BitMasks for TopologyHints\ntype BitMask interface {\n\tAdd(sockets ...int) error\n\tRemove(sockets ...int) error\n\tAnd(masks ...BitMask)\n\tOr(masks ...BitMask)\n\tClear()\n\tFill()\n\tIsEqual(mask BitMask) bool\n\tIsEmpty() bool\n\tIsSet(socket int) bool\n\tIsNarrowerThan(mask BitMask) bool\n\tString() string\n\tCount() int\n\tGetSockets() []int\n}\n\nfunc NewBitMask(sockets ...int) (BitMask, error) { ... }\n\npackage topologymanager\n\n// Manager interface provides methods for Kubelet to manage pod topology hints\ntype Manager interface {\n    // Implements pod admit handler interface\n    lifecycle.PodAdmitHandler\n    // Adds a hint provider to manager to indicate the hint provider\n    //wants to be consoluted when making topology hints\n    AddHintProvider(HintProvider)\n    // Adds pod to Manager for tracking\n    AddContainer(pod *v1.Pod, containerID string) error\n    // Removes pod from Manager tracking\n    RemoveContainer(containerID string) error\n    // Interface for storing pod topology hints\n    Store\n}\n\n// TopologyHint encodes locality to local resources. Each HintProvider provides\n// a list of these hints to the TopoologyManager for each container at pod\n// admission time.\ntype TopologyHint struct {\n    NUMANodeAffinity bitmask.BitMask\n    // Preferred is set to true when the BitMask encodes a preferred\n    // allocation for the Container. It is set to false otherwise.\n    Preferred bool\n}\n\n// HintProvider is implemented by Kubelet components that make\n// topology-related resource assignments. The Topology Manager consults each\n// hint provider at pod admission time.\ntype HintProvider interface {\n  // Returns hints if this hint provider has a preference; otherwise\n  // returns `nil` to indicate \"don't care\".\n  GetTopologyHints(pod v1.Pod, containerName string) map[string][]TopologyHint\n}\n\n// Store manages state related to the Topology Manager.\ntype Store interface {\n  // GetAffinity returns the preferred affinity as calculated by the\n  // TopologyManager across all hint providers for the supplied pod and\n  // container.\n  GetAffinity(podUID string, containerName string) TopologyHint\n}\n```\n\n_Listing: Topology Manager and related interfaces (sketch)._\n\n![topology-manager-components](https://user-images.githubusercontent.com/379372/47447523-8efd2b00-d772-11e8-924d-eea5a5e00037.png)\n\n_Figure: Topology Manager components._\n\n![topology-manager-instantiation](https://user-images.githubusercontent.com/379372/47447526-945a7580-d772-11e8-9761-5213d745e852.png)\n\n_Figure: Topology Manager instantiation and inclusion in pod admit lifecycle._\n \n### Feature Gate and Kubelet Flags\n \nA new feature gate will be added to enable the Topology Manager feature. This feature gate will be enabled in Kubelet, and will be disabled by default in the Alpha release.  \n\n * Proposed Feature Gate:  \n  `--feature-gate=TopologyManager=true`  \n \n This will be also followed by a Kubelet Flag for the Topology Manager Policy, which is described above. The `none` policy will be the default policy.\n \n * Proposed Policy Flag:  \n `--topology-manager-policy=none|best-effort|restricted|single-numa-node`  \n \n### Changes to Existing Components\n\n1. Kubelet consults Topology Manager for pod admission (discussed above.)\n1. Add two implementations of Topology Manager interface and a feature gate.\n    1. As much Topology Manager functionality as possible is stubbed when the\n       feature gate is disabled.\n    1. Add a functional Topology Manager that queries hint providers in order\n       to compute a preferred socket mask for each container.\n1. Add `GetTopologyHints()` method to CPU Manager.\n    1. CPU Manager static policy calls `GetAffinity()` method of\n       Topology Manager when deciding CPU affinity.\n1. Add `GetTopologyHints()` method to Device Manager.\n    1. Add `TopologyInfo` to Device structure in the device plugin\n       interface. Plugins should be able to determine the NUMA Node(s)\n       when enumerating supported devices. See the protocol diff below.\n    1. Device Manager calls `GetAffinity()` method of Topology Manager when\n       deciding device allocation.\n \n```diff\ndiff --git a/pkg/kubelet/apis/deviceplugin/v1beta1/api.proto b/pkg/kubelet/apis/deviceplugin/v1beta1/api.proto\nindex efbd72c133..f86a1a5512 100644\n+++ b/pkg/kubelet/apis/deviceplugin/v1beta1/api.proto\n@@ -73,6 +73,10 @@ message ListAndWatchResponse {\n \trepeated Device devices = 1;\n }\n\n+message TopologyInfo {\n+  repeated NUMANode nodes = 1;\n+}\n+\n+message NUMANode {\n+    int64 ID = 1;\n+}\n+\n /* E.g:\n * struct Device {\n *    ID: \"GPU-fef8089b-4820-abfc-e83e-94318197576e\",\n *    State: \"Healthy\",\n+ *    Topology: \n+ *      Nodes: \n+ *        ID: 1 \n@@ -85,6 +89,8 @@ message Device {\n \tstring ID = 1;\n \t// Health of the device, can be healthy or unhealthy, see constants.go\n \tstring health = 2;\n+\t// Topology details of the device\n+\tTopologyInfo topology = 3;\n }\n```\n\n_Listing: Amended device plugin gRPC protocol._\n\n![topology-manager-wiring](https://user-images.githubusercontent.com/379372/47447533-9a505680-d772-11e8-95ca-ef9a8290a46a.png)\n\n_Figure: Topology Manager hint provider registration._\n\n![topology-manager-hints](https://user-images.githubusercontent.com/379372/47447543-a0463780-d772-11e8-8412-8bf4a0571513.png)\n\n_Figure: Topology Manager fetches affinity from hint providers._\n\n# Graduation Criteria\n\n## Phase 1: Alpha (v1.16) [COMPLETED]\n\n* Feature gate is disabled by default.\n* Alpha-level documentation.\n* Unit test coverage.\n* CPU Manager allocation policy takes topology hints into account.\n* Device plugin interface includes NUMA Node ID.\n* Device Manager allocation policy takes topology hints into account.\n\n## Phase 2: Beta (target v1.17)\n\n* Enable the feature gate by default.\n* Provide beta-level documentation.\n* Add node E2E tests.\n* Allow pods in all QoS classes to request aligned resources.\n* Guarantee aligned resources for multiple containers in a pod.\n* Refactor to easily support different merge strategies for different policies.\n* Add support for device-specific topology constraints beyond NUMA.\n\n## GA (stable)\n\n* Support hugepages alignment.\n* User feedback.\n* *TBD*\n\n# Challenges\n\n* Testing the Topology Manager in a continuous integration environment\n  depends on cloud infrastructure to expose multi-node topologies\n  to guest virtual machines.\n* Implementing the `GetHints()` interface may prove challenging.\n\n# Limitations\n\n* Alignment is only possible for pods in the `Guaranteed` QoS class.\n* Alignment is only guaranteed for a single container inside a pod.\n\n# Alternatives\n\n* [AutoNUMA][numa-challenges]: This kernel feature affects memory\n  allocation and thread scheduling, but does not address device locality.\n\n# References\n\n* *TBD*\n\n[k8s-issue-49964]: https://github.com/kubernetes/kubernetes/issues/49964\n[nfd-issue-84]: https://github.com/kubernetes-incubator/node-feature-discovery/issues/84\n[sriov-issue-10]: https://github.com/hustcat/sriov-cni/issues/10\n[proposal-affinity]: https://github.com/kubernetes/community/pull/171\n[numa-challenges]: https://queue.acm.org/detail.cfm?id=2852078\n"
+  },
+  {
+    "id": "608d8d959db2431da4008c580edf37e7",
+    "title": "Quotas for Ephemeral Storage",
+    "authors": ["@RobertKrawitz"],
+    "owningSig": "sig-node",
+    "participatingSigs": ["sig-node"],
+    "reviewers": ["@dashpole", "@derekwaynecarr"],
+    "approvers": ["@dchen1107", "@derekwaynecarr"],
+    "editor": "TBD",
+    "creationDate": "2018-09-06",
+    "lastUpdated": "2019-06-04",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Quotas for Ephemeral Storage\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n  - [Project Quotas](#project-quotas)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n  - [Future Work](#future-work)\n- [Proposal](#proposal)\n  - [Control over Use of Quotas](#control-over-use-of-quotas)\n  - [Operation Notes](#operation-notes)\n    - [Selecting a Project ID](#selecting-a-project-id)\n    - [Determine Whether a Project ID Applies To a Directory](#determine-whether-a-project-id-applies-to-a-directory)\n    - [Return a Project ID To the System](#return-a-project-id-to-the-system)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n    - [Implementation Strategy](#implementation-strategy)\n      - [Future](#future)\n    - [Notes on Implementation](#notes-on-implementation)\n    - [Notes on Code Changes](#notes-on-code-changes)\n    - [Testing Strategy](#testing-strategy)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n  - [Phase 1: Alpha (1.15)](#phase-1-alpha-115)\n  - [Phase 2: Beta (target 1.16)](#phase-2-beta-target-116)\n  - [Phase 3: GA](#phase-3-ga)\n- [Performance Benchmarks](#performance-benchmarks)\n  - [Elapsed Time](#elapsed-time)\n  - [User CPU Time](#user-cpu-time)\n  - [System CPU Time](#system-cpu-time)\n- [Implementation History](#implementation-history)\n  - [Version 1.15](#version-115)\n- [Drawbacks [optional]](#drawbacks-optional)\n- [Alternatives [optional]](#alternatives-optional)\n  - [Alternative quota-based implementation](#alternative-quota-based-implementation)\n  - [Alternative loop filesystem-based implementation](#alternative-loop-filesystem-based-implementation)\n- [Infrastructure Needed [optional]](#infrastructure-needed-optional)\n- [References](#references)\n  - [Bugs Opened Against Filesystem Quotas](#bugs-opened-against-filesystem-quotas)\n    - [CVE](#cve)\n    - [Other Security Issues Without CVE](#other-security-issues-without-cve)\n  - [Other Linux Quota-Related Bugs Since 2012](#other-linux-quota-related-bugs-since-2012)\n\u003c!-- /toc --\u003e\n\n[Tools for generating]: https://github.com/ekalinin/github-markdown-toc\n\n## Summary\n\nThis proposal applies to the use of quotas for ephemeral-storage\nmetrics gathering.  Use of quotas for ephemeral-storage limit\nenforcement is a [non-goal](#non-goals), but as the architecture and\ncode will be very similar, there are comments interspersed related to\nenforcement.  _These comments will be italicized_.\n\nLocal storage capacity isolation, aka ephemeral-storage, was\nintroduced into Kubernetes via\n\u003chttps://github.com/kubernetes/features/issues/361\u003e.  It provides\nsupport for capacity isolation of shared storage between pods, such\nthat a pod can be limited in its consumption of shared resources and\ncan be evicted if its consumption of shared storage exceeds that\nlimit.  The limits and requests for shared ephemeral-storage are\nsimilar to those for memory and CPU consumption.\n\nThe current mechanism relies on periodically walking each ephemeral\nvolume (emptydir, logdir, or container writable layer) and summing the\nspace consumption.  This method is slow, can be fooled, and has high\nlatency (i. e. a pod could consume a lot of storage prior to the\nkubelet being aware of its overage and terminating it).\n\nThe mechanism proposed here utilizes filesystem project quotas to\nprovide monitoring of resource consumption _and optionally enforcement\nof limits._  Project quotas, initially in XFS and more recently ported\nto ext4fs, offer a kernel-based means of monitoring _and restricting_\nfilesystem consumption that can be applied to one or more directories.\n\nA prototype is in progress; see \u003chttps://github.com/kubernetes/kubernetes/pull/66928\u003e.\n\n### Project Quotas\n\nProject quotas are a form of filesystem quota that apply to arbitrary\ngroups of files, as opposed to file user or group ownership.  They\nwere first implemented in XFS, as described here:\n\u003chttp://xfs.org/docs/xfsdocs-xml-dev/XFS_User_Guide/tmp/en-US/html/xfs-quotas.html\u003e.\n\nProject quotas for ext4fs were [proposed in late\n2014](https://lwn.net/Articles/623835/) and added to the Linux kernel\nin early 2016, with\ncommit\n[391f2a16b74b95da2f05a607f53213fc8ed24b8e](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=391f2a16b74b95da2f05a607f53213fc8ed24b8e).\nThey were designed to be compatible with XFS project quotas.\n\nEach inode contains a 32-bit project ID, to which optionally quotas\n(hard and soft limits for blocks and inodes) may be applied.  The\ntotal blocks and inodes for all files with the given project ID are\nmaintained by the kernel.  Project quotas can be managed from\nuserspace by means of the `xfs_quota(8)` command in foreign filesystem\n(`-f`) mode; the traditional Linux quota tools do not manipulate\nproject quotas.  Programmatically, they are managed by the `quotactl(2)`\nsystem call, using in part the standard quota commands and in part the\nXFS quota commands; the man page implies incorrectly that the XFS\nquota commands apply only to XFS filesystems.\n\nThe project ID applied to a directory is inherited by files created\nunder it.  Files cannot be (hard) linked across directories with\ndifferent project IDs.  A file's project ID cannot be changed by a\nnon-privileged user, but a privileged user may use the `xfs_io(8)`\ncommand to change the project ID of a file.\n\nFilesystems using project quotas may be mounted with quotas either\nenforced or not; the non-enforcing mode tracks usage without enforcing\nit.  A non-enforcing project quota may be implemented on a filesystem\nmounted with enforcing quotas by setting a quota too large to be hit.\nThe maximum size that can be set varies with the filesystem; on a\n64-bit filesystem it is 2^63-1 bytes for XFS and 2^58-1 bytes for\next4fs.\n\nConventionally, project quota mappings are stored in `/etc/projects` and\n`/etc/projid`; these files exist for user convenience and do not have\nany direct importance to the kernel.  `/etc/projects` contains a mapping\nfrom project ID to directory/file; this can be a one to many mapping\n(the same project ID can apply to multiple directories or files, but\nany given directory/file can be assigned only one project ID).\n`/etc/projid` contains a mapping from named projects to project IDs.\n\nThis proposal utilizes hard project quotas for both monitoring _and\nenforcement_.  Soft quotas are of no utility; they allow for temporary\noverage that, after a programmable period of time, is converted to the\nhard quota limit.\n\n\n## Motivation\n\nThe mechanism presently used to monitor storage consumption involves\nuse of `du` and `find` to periodically gather information about\nstorage and inode consumption of volumes.  This mechanism suffers from\na number of drawbacks:\n\n* It is slow.  If a volume contains a large number of files, walking\n  the directory can take a significant amount of time.  There has been\n  at least one known report of nodes becoming not ready due to volume\n  metrics: \u003chttps://github.com/kubernetes/kubernetes/issues/62917\u003e\n* It is possible to conceal a file from the walker by creating it and\n  removing it while holding an open file descriptor on it.  POSIX\n  behavior is to not remove the file until the last open file\n  descriptor pointing to it is removed.  This has legitimate uses; it\n  ensures that a temporary file is deleted when the processes using it\n  exit, and it minimizes the attack surface by not having a file that\n  can be found by an attacker.  The following pod does this; it will\n  never be caught by the present mechanism:\n\n```yaml\napiVersion: v1\nkind: Pod\nmax:\nmetadata:\n  name: \"diskhog\"\nspec:\n  containers:\n  - name: \"perl\"\n    resources:\n      limits:\n        ephemeral-storage: \"2048Ki\"\n    image: \"perl\"\n    command:\n    - perl\n    - -e\n    - \u003e\n      my $file = \"/data/a/a\"; open OUT, \"\u003e$file\" or die \"Cannot open $file: $!\\n\"; unlink \"$file\" or die \"cannot unlink $file: $!\\n\"; my $a=\"0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\"; foreach my $i (0..200000000) { print OUT $a; }; sleep 999999\n    volumeMounts:\n    - name: a\n      mountPath: /data/a\n  volumes:\n  - name: a\n    emptyDir: {}\n```\n* It is reactive rather than proactive.  It does not prevent a pod\n  from overshooting its limit; at best it catches it after the fact.\n  On a fast storage medium, such as NVMe, a pod may write 50 GB or\n  more of data before the housekeeping performed once per minute\n  catches up to it.  If the primary volume is the root partition, this\n  will completely fill the partition, possibly causing serious\n  problems elsewhere on the system.  This proposal does not address\n  this issue; _a future enforcing project would_.\n\nIn many environments, these issues may not matter, but shared\nmulti-tenant environments need these issues addressed.\n\n### Goals\n\nThese goals apply only to local ephemeral storage, as described in\n\u003chttps://github.com/kubernetes/features/issues/361\u003e.\n\n* Primary: improve performance of monitoring by using project quotas\n  in a non-enforcing way to collect information about storage\n  utilization of ephemeral volumes.\n* Primary: detect storage used by pods that is concealed by deleted\n  files being held open.\n* Primary: this will not interfere with the more common user and group\n  quotas.\n\n### Non-Goals\n\n* Application to storage other than local ephemeral storage.\n* Application to container copy on write layers.  That will be managed\n  by the container runtime.  For a future project, we should work with\n  the runtimes to use quotas for their monitoring.\n* Elimination of eviction as a means of enforcing ephemeral-storage\n  limits.  Pods that hit their ephemeral-storage limit will still be\n  evicted by the kubelet even if their storage has been capped by\n  enforcing quotas.\n* Enforcing node allocatable (limit over the sum of all pod's disk\n  usage, including e. g. images).\n* Enforcing limits on total pod storage consumption by any means, such\n  that the pod would be hard restricted to the desired storage limit.\n\n### Future Work\n\n* _Enforce limits on per-volume storage consumption by using\n  enforced project quotas._\n\n## Proposal\n\nThis proposal applies project quotas to emptydir volumes on qualifying\nfilesystems (ext4fs and xfs with project quotas enabled).  Project\nquotas are applied by selecting an unused project ID (a 32-bit\nunsigned integer), setting a limit on space and/or inode consumption,\nand attaching the ID to one or more files.  By default (and as\nutilized herein), if a project ID is attached to a directory, it is\ninherited by any files created under that directory.\n\n_If we elect to use the quota as enforcing, we impose a quota\nconsistent with the desired limit._  If we elect to use it as\nnon-enforcing, we impose a large quota that in practice cannot be\nexceeded (2^63-1 bytes for XFS, 2^58-1 bytes for ext4fs).\n\nFor discussion of the low level implementation strategy, [see\nbelow](#implementation-detailsnotesconstraints-optional).\n\n### Control over Use of Quotas\n\nAt present, two feature gates control operation of quotas:\n\n* `LocalStorageCapacityIsolation` must be enabled for any use of\n  quotas.\n\n* `LocalStorageCapacityIsolationFSQuotaMonitoring` must be enabled in addition.  If this is\n  enabled, quotas are used for monitoring, but not enforcement.  At\n  present, this defaults to False, but the intention is that this will\n  default to True by initial release.\n\n### Operation Flow -- Applying a Quota\n\n* Caller (emptydir volume manager or container runtime) creates an\n  emptydir volume, with an empty directory at a location of its\n  choice.\n* Caller requests that a quota be applied to a directory.\n* Determine whether a quota can be imposed on the directory, by asking\n  each quota provider (one per filesystem type) whether it can apply a\n  quota to the directory.  If no provider claims the directory, an\n  error status is returned to the caller.\n* Select an unused project ID ([see below](#selecting-a-project-id)).\n* Set the desired limit on the project ID, in a filesystem-dependent\n  manner ([see below](#notes-on-implementation)).\n* Apply the project ID to the directory in question, in a\n  filesystem-dependent manner.\n\nAn error at any point results in no quota being applied and no change\nto the state of the system.  The caller in general should not assume a\npriori that the attempt will be successful.  It could choose to reject\na request if a quota cannot be applied, but at this time it will\nsimply ignore the error and proceed as today.\n\n### Operation Flow -- Retrieving Storage Consumption\n\n* Caller (kubelet metrics code, cadvisor, container runtime) asks the\n  quota code to compute the amount of storage used under the\n  directory.\n* Determine whether a quota applies to the directory, in a\n  filesystem-dependent manner ([see below](#notes-on-implementation)).\n* If so, determine how much storage or how many inodes are utilized,\n  in a filesystem dependent manner.\n\nIf the quota code is unable to retrieve the consumption, it returns an\nerror status and it is up to the caller to utilize a fallback\nmechanism (such as the directory walk performed today).\n\n### Operation Flow -- Removing a Quota.\n\n* Caller requests that the quota be removed from a directory.\n* Determine whether a project quota applies to the directory.\n* Remove the limit from the project ID associated with the directory.\n* Remove the association between the directory and the project ID.\n* Return the project ID to the system to allow its use elsewhere ([see\n  below](#return-a-project-id-to-the-system)).\n* Caller may delete the directory and its contents (normally it will).\n\n### Operation Notes\n\n#### Selecting a Project ID\n\nProject IDs are a shared space within a filesystem.  If the same\nproject ID is assigned to multiple directories, the space consumption\nreported by the quota will be the sum of that of all of the\ndirectories.  Hence, it is important to ensure that each directory is\nassigned a unique project ID (unless it is desired to pool the storage\nuse of multiple directories).\n\nThe canonical mechanism to record persistently that a project ID is\nreserved is to store it in the `/etc/projid` (`projid[5]`) and/or\n`/etc/projects` (`projects(5)`) files.  However, it is possible to utilize\nproject IDs without recording them in those files; they exist for\nadministrative convenience but neither the kernel nor the filesystem\nis aware of them.  Other ways can be used to determine whether a\nproject ID is in active use on a given filesystem:\n\n* The quota values (in blocks and/or inodes) assigned to the project\n  ID are non-zero.\n* The storage consumption (in blocks and/or inodes) reported under the\n  project ID are non-zero.\n\nThe algorithm to be used is as follows:\n\n* Lock this instance of the quota code against re-entrancy.\n* open and `flock()` the `/etc/project` and `/etc/projid` files, so that\n  other uses of this code are excluded.\n* Start from a high number (the prototype uses 1048577).\n* Iterate from there, performing the following tests:\n   * Is the ID reserved by this instance of the quota code?\n   * Is the ID present in `/etc/projects`?\n   * Is the ID present in `/etc/projid`?\n   * Are the quota values and/or consumption reported by the kernel\n     non-zero?  This test is restricted to 128 iterations to ensure\n     that a bug here or elsewhere does not result in an infinite loop\n     looking for a quota ID.\n* If an ID has been found:\n   * Add it to an in-memory copy of `/etc/projects` and `/etc/projid` so\n     that any other uses of project quotas do not reuse it.\n   * Write temporary copies of `/etc/projects` and `/etc/projid` that are\n     `flock()`ed\n   * If successful, rename the temporary files appropriately (if\n     rename of one succeeds but the other fails, we have a problem\n     that we cannot recover from, and the files may be inconsistent).\n* Unlock `/etc/projid` and `/etc/projects`.\n* Unlock this instance of the quota code.\n\nA minor variation of this is used if we want to reuse an existing\nquota ID.\n\n#### Determine Whether a Project ID Applies To a Directory\n\nIt is possible to determine whether a directory has a project ID\napplied to it by requesting (via the `lsattr(1)` command) the project\nID associated with the directory.  Whie the specifics are\nfilesystem-dependent, the basic method is the same for at least XFS\nand ext4fs.\n\nIt is not possible to determine in constant operations the directory\nor directories to which a project ID is applied.  It is possible to\ndetermine whether a given project ID has been applied to an existing\ndirectory or files (although those will not be known); the reported\nconsumption will be non-zero.\n\nThe code records internally the project ID applied to a directory, but\nit cannot always rely on this.  In particular, if the kubelet has\nexited and has been restarted (and hence the quota applying to the\ndirectory should be removed), the map from directory to project ID is\nlost.  If it cannot find a map entry, it falls back on the approach\ndiscussed above.\n\n#### Return a Project ID To the System\n\nThe algorithm used to return a project ID to the system is very\nsimilar to the algorithm used to select a project ID, except of course\nfor selecting a project ID.  It performs the same sequence of locking\n`/etc/project` and `/etc/projid`, editing a copy of the file, and\nrestoring it.\n\nIf the project ID is applied to multiple directories and the code can\ndetermine that, it will not remove the project ID from `/etc/projid`\nuntil the last reference is removed.  While it is not anticipated in\nthis KEP that this mode of operation will be used, at least initially,\nthis can be detected even on kubelet restart by looking at the\nreference count in `/etc/projects`.\n\n### Implementation Details/Notes/Constraints [optional]\n\n#### Implementation Strategy\n\nThe initial implementation will be done by shelling out to the\n`xfs_quota(8)` and `lsattr(1)` commands to manipulate quotas.  It is\npossible to use the `quotactl(2)` and `ioctl(2)` system calls to do\nthis, but the `quotactl(2)` system call is not supported by Go at all,\nand `ioctl(2)` subcommands required for manipulating quotas are only\npartially supported.  Use of these system calls would require either\nuse of [cgo](https://golang.org/cmd/cgo/), which is not supported in\nKubernetes, or copying the necessary structure definitions and\nconstants into the source.\n\nThe use of Linux commands rather than system calls likely poses some\nefficiency issues, but these commands are only issued when ephemeral\nvolumes are created and destroyed, or during monitoring, when they are\ncalled once per minute.  At present, monitoring is done by shelling\nout to the `du(1)` and `find(1)` commands, which are much less\nefficient, as they perform filesystem scans.  *The performance of\nthese commands must be measured under load prior to making this\nfeature used by default.*\n\nAll `xfs_quota(8)` commands are invoked as\n\n`xfs_quota -t \u003ctmp_mounts_file\u003e -P/dev/null -D/dev/null -f \u003cmountpoint\u003e -c \u003ccommand\u003e`\n\nThe -P and -D arguments tell xfs_quota not to read the usual\n`/etc/projid` and `/etc/projects` files, and to use the empty special\nfile `/dev/null` as stand-ins.  `xfs_quota` reads the projid and\nprojects files and attempts to access every listed mountpoint for\nevery command.  As we use numeric project IDs for all purposes, it is\nnot necessary to incur this overhead.\n\nThe `-t` argument is to mitigate a hazard of `xfs_quota(8)` in the\npresence of stuck NFS (or similar) mounts.  By default, `xfs_quota(8)`\n`stat(2)`s every mountpoint on the system, which could hang on a stuck\nmount.  The `-t` option is used to pass in a temporary mounts file\ncontaining only the filesystem we care about.\n\nThe following operations are performed, with the listed commands\n(using `xfs_quota(8)` except as noted)\n\n* Determine whether quotas are enabled on the specified filesystem\n\n  `state -p`\n\n* Apply a limit to a project ID (note that at present the largest\n  possible limit is applied, allowing the quota system to be used for\n  monitoring only)\n\n  `limit -p bhard=\u003cblocks\u003e bsoft=\u003cblocks\u003e \u003cprojectID\u003e`\n\n* Apply a project ID to a directory, enabling a quota on that\n  directory\n\n  `project -s -p \u003cdirectory\u003e \u003cprojectID\u003e`\n\n* Retrieve the number of blocks used by a given project ID\n\n  `quota -p -N -n -v -b \u003cprojectID\u003e`\n\n* Retrieve the number of inodes used by a given project ID\n\n  `quota -p -N -n -v -n \u003cprojectID\u003e`\n\n* Determine whether a specified directory has a quota ID applied to\n  it, and if so, what that ID is (using `lsattr(1)`)\n\n  `lsattr -pd \u003cpath\u003e`\n\n##### Future\n\nIn the long run, we should work to add the necessary constructs to the\nGo language, allowing direct use of the necessary system calls without\nthe use of cgo.\n\n\n#### Notes on Implementation\n\nThe primary new interface defined is the quota interface in\n`pkg/volume/util/quota/quota.go`.  This defines five operations:\n\n* Does the specified directory support quotas?\n\n* Assign a quota to a directory.  If a non-empty pod UID is provided,\n  the quota assigned is that of any other directories under this pod\n  UID; if an empty pod UID is provided, a unique quota is assigned.\n\n* Retrieve the consumption of the specified directory.  If the quota\n  code cannot handle it efficiently, it returns an error and the\n  caller falls back on existing mechanism.\n\n* Retrieve the inode consumption of the specified directory; same\n  description as above.\n\n* Remove quota from a directory.  If a non-empty pod UID is passed, it\n  is checked against that recorded in-memory (if any).  The quota is\n  removed from the specified directory.  This can be used even if\n  AssignQuota has not been used; it inspects the directory and removes\n  the quota from it.  This permits stale quotas from an interrupted\n  kubelet to be cleaned up.\n\nTwo implementations are provided: `quota_linux.go` (for Linux) and\n`quota_unsupported.go` (for other operating systems).  The latter\nreturns an error for all requests.\n\nAs the quota mechanism is intended to support multiple filesystems,\nand different filesystems require different low level code for\nmanipulating quotas, a provider is supplied that finds an appropriate\nquota applier implementation for the filesystem in question.  The low\nlevel quota applier provides similar operations to the top level quota\ncode, with two exceptions:\n\n* No operation exists to determine whether a quota can be applied\n  (that is handled by the provider).\n\n* An additional operation is provided to determine whether a given\n  quota ID is in use within the filesystem (outside of `/etc/projects`\n  and `/etc/projid`).\n\nThe two quota providers in the initial implementation are in\n`pkg/volume/util/quota/extfs` and `pkg/volume/util/quota/xfs`.  While\nsome quota operations do require different system calls, a lot of the\ncode is common, and factored into\n`pkg/volume/util/quota/common/quota_linux_common_impl.go`.\n\n#### Notes on Code Changes\n\nThe prototype for this project is mostly self-contained within\n`pkg/volume/util/quota` and a few changes to\n`pkg/volume/empty_dir/empty_dir.go`.  However, a few changes were\nrequired elsewhere:\n\n* The operation executor needs to pass the desired size limit to the\n  volume plugin where appropriate so that the volume plugin can impose\n  a quota.  The limit is passed as 0 (do not use quotas), _positive\n  number (impose an enforcing quota if possible, measured in bytes),_\n  or -1 (impose a non-enforcing quota, if possible) on the volume.\n\n  This requires changes to\n  `pkg/volume/util/operationexecutor/operation_executor.go` (to add\n  `DesiredSizeLimit` to `VolumeToMount`),\n  `pkg/kubelet/volumemanager/cache/desired_state_of_world.go`, and\n  `pkg/kubelet/eviction/helpers.go` (the latter in order to determine\n  whether the volume is a local ephemeral one).\n\n* The volume manager (in `pkg/volume/volume.go`) changes the\n  `Mounter.SetUp` and `Mounter.SetUpAt` interfaces to take a new\n  `MounterArgs` type rather than an `FsGroup` (`*int64`).  This is to\n  allow passing the desired size and pod UID (in the event we choose\n  to implement quotas shared between multiple volumes; [see\n  below](#alternative-quota-based-implementation)).  This required\n  small changes to all volume plugins and their tests, but will in the\n  future allow adding additional data without having to change code\n  other than that which uses the new information.\n\n#### Testing Strategy\n\nThe quota code is by an large not very amendable to unit tests.  While\nthere are simple unit tests for parsing the mounts file, and there\ncould be tests for parsing the projects and projid files, the real\nwork (and risk) involves interactions with the kernel and with\nmultiple instances of this code (e. g. in the kubelet and the runtime\nmanager, particularly under stress).  It also requires setup in the\nform of a prepared filesystem.  It would be better served by\nappropriate end to end tests.\n\n### Risks and Mitigations\n\n* The SIG raised the possibility of a container being unable to exit\n  should we enforce quotas, and the quota interferes with writing the\n  log.  This can be mitigated by either not applying a quota to the\n  log directory and using the du mechanism, or by applying a separate\n  non-enforcing quota to the log directory.\n\n  As log directories are write-only by the container, and consumption\n  can be limited by other means (as the log is filtered by the\n  runtime), I do not consider the ability to write uncapped to the log\n  to be a serious exposure.\n\n  Note in addition that even without quotas it is possible for writes\n  to fail due to lack of filesystem space, which is effectively (and\n  in some cases operationally) indistinguishable from exceeding quota,\n  so even at present code must be able to handle those situations.\n\n* Filesystem quotas may impact performance to an unknown degree.\n  Information on that is hard to come by in general, and one of the\n  reasons for using quotas is indeed to improve performance.  If this\n  is a problem in the field, merely turning off quotas (or selectively\n  disabling project quotas) on the filesystem in question will avoid\n  the problem.  Against the possibility that cannot be done\n  (because project quotas are needed for other purposes), we should\n  provide a way to disable use of quotas altogether via a feature\n  gate.\n\n  A report \u003chttps://blog.pythonanywhere.com/110/\u003e notes that an\n  unclean shutdown on Linux kernel versions between 3.11 and 3.17 can\n  result in a prolonged downtime while quota information is restored.\n  Unfortunately, [the link referenced\n  here](http://oss.sgi.com/pipermail/xfs/2015-March/040879.html) is no\n  longer available.\n\n* Bugs in the quota code could result in a variety of regression\n  behavior.  For example, if a quota is incorrectly applied it could\n  result in ability to write no data at all to the volume.  This could\n  be mitigated by use of non-enforcing quotas.  XFS in particular\n  offers the `pqnoenforce` mount option that makes all quotas\n  non-enforcing.\n\n\n## Graduation Criteria\n\nThe following criteria applies to\n`LocalStorageCapacityIsolationFSMonitoring`:\n\n### Phase 1: Alpha (1.15)\n\n- Support integrated in kubelet\n- Alpha-level documentation\n- Unit test coverage\n- Node e2e test\n\n### Phase 2: Beta (target 1.16)\n\n- User feedback\n- Benchmarks to determine latency and overhead of using quotas\n  relative to existing monitoring solution\n- Cleanup\n\n### Phase 3: GA\n\n- TBD\n\n## Performance Benchmarks\n\nI performed a microbenchmark consisting of various operations on a\ndirectory containing 4096 subdirectories each containing 2048 1Kbyte\nfiles.  The operations performed were as follows, in sequence:\n\n* *Create Files*: Create 4K directories each containing 2K files as\n  described, in depth-first order.\n  \n* *du*: run `du` immediately after creating the files.\n\n* *quota*: where applicable, run `xfs_quota` immediately after `du`.\n\n* *du (repeat)*: repeat the `du` invocation.\n\n* *quota (repeat)*: repeat the `xfs_quota` invocation.\n\n* *du (after remount)*: run `mount -o remount \u003cfilesystem\u003e`\n  immediately followed by `du`.\n  \n* *quota (after remount)*: run `mount -o remount \u003cfilesystem\u003e`\n  immediately followed by `xfs_quota`.\n  \n* *unmount*: `umount` the filesystem.\n\n* *mount*: `mount` the filesystem.\n\n* *quota after umount/mount*: run `xfs_quota` after unmounting and\n  mounting the filesystem.\n\n* *du after umount/mount*: run `du` after unmounting and\n  mounting the filesystem.\n  \n* *Remove Files*: remove the test files.\n\nThe test was performed on four separate filesystems:\n\n* XFS filesystem, with quotas enabled (256 GiB, 128 Mi inodes)\n* XFS filesystem, with quotas disabled (64 GiB, 32 Mi inodes)\n* ext4fs filesystem, with quotas enabled (250 GiB, 16 Mi inodes)\n* ext4fs filesystem, with quotas disabled (60 GiB, 16 Mi inodes)\n\nOther notes:\n\n* All filesystems reside on an otherwise idle HP EX920 1TB NVMe on a Lenovo ThinkPad P50 with 64 GB RAM and Intel Core i7-6820HQ CPU (4 cores/8 threads total) running Fedora 30 (5.1.5-300.fc30.x86_64)\n* Five runs were conducted with each combination; the median value is used.  All times are in seconds.\n* Space consumption was calculated with the filesystem hot (after files were created), warm (after `mount -o remount`), and cold (after umount/mount of the filesystem).\n* Note that in all cases xfs_quota consumed zero time as reported by `/usr/bin/time`.\n* User and system time are not available for file creation.\n* All calls to `xfs_quota` and `mount` consumed less than 0.01 seconds elapsed, user, and CPU time and are not reported here.\n* Removing files was consistently faster with quotas disabled than with quotas enabled.  With ext4fs, du was faster with quotas disabled than with quotas enabled.  With XFS, creating files may have been faster with quotas disabled than with quotas enabled, but the difference was small.  In other cases, the difference was within noise.\n\n### Elapsed Time\n\n| *Operation*                | *XFS+Quota* | *XFS*   | *Ext4fs+Quota* | *Ext4fs* |\n| Create Files             | 435.0     | 419.0 | 348.0        | 343.0  |\n| du                       | 12.1      | 12.6  | 14.3         | 14.3   |\n| du (repeat)              | 12.0      | 12.1  | 14.1         | 14.0   |\n| du (after remount)       | 23.2      | 23.2  | 39.0         | 24.6   |\n| unmount                  | 12.2      | 12.1  | 9.8          | 9.8    |\n| du after umount/mount    | 103.6     | 138.8 | 40.2         | 38.8   |\n| Remove Files             | 196.0     | 159.8 | 105.2        | 90.4   |\n\n### User CPU Time\n\nAll calls to `umount` consumed less than 0.01 second of user CPU time\nand are not reported here.\n\n| *Operation*                | *XFS+Quota* | *XFS*   | *Ext4fs+Quota* | *Ext4fs* |\n| du                       | 3.7       | 3.7   | 3.7          | 3.7    |\n| du (repeat)              | 3.7       | 3.7   | 3.7          | 3.8    |\n| du (after remount)       | 3.3       | 3.3   | 3.7          | 3.6    |\n| du after umount/mount    | 8.1       | 10.2  | 3.9          | 3.7    |\n| Remove Files             | 4.3       | 4.1   | 4.2          | 4.3    |\n\n### System CPU Time\n\n| *Operation*                | *XFS+Quota* | *XFS*   | *Ext4fs+Quota* | *Ext4fs* |\n| du                       | 8.3       | 8.6   | 10.5         | 10.5   |\n| du (repeat)              | 8.3       | 8.4   | 10.4         | 10.4   |\n| du (after remount)       | 19.8      | 19.8  | 28.8         | 20.9   |\n| unmount                  | 10.2      | 10.1  | 8.1          | 8.1    |\n| du after umount/mount    | 66.0      | 82.4  | 29.2         | 28.1   |\n| Remove Files             | 188.6     | 156.6 | 90.4         | 81.8   |\n\n## Implementation History\n\n### Version 1.15\n\n` LocalStorageCapacityIsolationFSMonitoring` implemented at Alpha\n\n## Drawbacks [optional]\n\n* Use of quotas, particularly the less commonly used project quotas,\n  requires additional action on the part of the administrator.  In\n  particular:\n   * ext4fs filesystems must be created with additional options that\n     are not enabled by default:\n```\nmkfs.ext4 -O quota,project -E quotatype=usrquota:grpquota:prjquota _device_\n```\n   * An additional option (`prjquota`) must be applied in `/etc/fstab`\n   * If the root filesystem is to be quota-enabled, it must be set in\n     the grub options.\n* Use of project quotas for this purpose will preclude future use\n  within containers.\n\n## Alternatives [optional]\n\nI have considered two classes of alternatives:\n\n* Alternatives based on quotas, with different implementation\n\n* Alternatives based on loop filesystems without use of quotas\n\n### Alternative quota-based implementation\n\nWithin the basic framework of using quotas to monitor and potentially\nenforce storage utilization, there are a number of possible options:\n\n* Utilize per-volume non-enforcing quotas to monitor storage (the\n  first stage of this proposal).\n\n  This mostly preserves the current behavior, but with more efficient\n  determination of storage utilization and the possibility of building\n  further on it.  The one change from current behavior is the ability\n  to detect space used by deleted files.\n\n* Utilize per-volume enforcing quotas to monitor and enforce storage\n  (the second stage of this proposal).\n\n  This allows partial enforcement of storage limits.  As local storage\n  capacity isolation works at the level of the pod, and we have no\n  control of user utilization of ephemeral volumes, we would have to\n  give each volume a quota of the full limit.  For example, if a pod\n  had a limit of 1 MB but had four ephemeral volumes mounted, it would\n  be possible for storage utilization to reach (at least temporarily)\n  4MB before being capped.\n\n* Utilize per-pod enforcing user or group quotas to enforce storage\n  consumption, and per-volume non-enforcing quotas for monitoring.\n\n  This would offer the best of both worlds: a fully capped storage\n  limit combined with efficient reporting.  However, it would require\n  each pod to run under a distinct UID or GID.  This may prevent pods\n  from using setuid or setgid or their variants, and would interfere\n  with any other use of group or user quotas within Kubernetes.\n\n* Utilize per-pod enforcing quotas to monitor and enforce storage.\n\n  This allows for full enforcement of storage limits, at the expense\n  of being able to efficiently monitor per-volume storage\n  consumption.  As there have already been reports of monitoring\n  causing trouble, I do not advise this option.\n\n  A variant of this would report (1/N) storage for each covered\n  volume, so with a pod with a 4MiB quota and 1MiB total consumption,\n  spread across 4 ephemeral volumes, each volume would report a\n  consumption of 256 KiB.  Another variant would change the API to\n  report statistics for all ephemeral volumes combined.  I do not\n  advise this option.\n\n### Alternative loop filesystem-based implementation\n\nAnother way of isolating storage is to utilize filesystems of\npre-determined size, using the loop filesystem facility within Linux.\nIt is possible to create a file and run `mkfs(8)` on it, and then to\nmount that filesystem on the desired directory.  This both limits the\nstorage available within that directory and enables quick retrieval of\nit via `statfs(2)`.\n\nCleanup of such a filesystem involves unmounting it and removing the\nbacking file.\n\nThe backing file can be created as a sparse file, and the `discard`\noption can be used to return unused space to the system, allowing for\nthin provisioning.\n\nI conducted preliminary investigations into this.  While at first it\nappeared promising, it turned out to have multiple critical flaws:\n\n* If the filesystem is mounted without the `discard` option, it can\n  grow to the full size of the backing file, negating any possibility\n  of thin provisioning.  If the file is created dense in the first\n  place, there is never any possibility of thin provisioning without\n  use of `discard`.\n\n  If the backing file is created densely, it additionally may require\n  significant time to create if the ephemeral limit is large.\n\n* If the filesystem is mounted `nosync`, and is sparse, it is possible\n  for writes to succeed and then fail later with I/O errors when\n  synced to the backing storage.  This will lead to data corruption\n  that cannot be detected at the time of write.\n\n  This can easily be reproduced by e. g. creating a 64MB filesystem\n  and within it creating a 128MB sparse file and building a filesystem\n  on it.  When that filesystem is in turn mounted, writes to it will\n  succeed, but I/O errors will be seen in the log and the file will be\n  incomplete:\n\n```\n# mkdir /var/tmp/d1 /var/tmp/d2\n# dd if=/dev/zero of=/var/tmp/fs1 bs=4096 count=1 seek=16383\n# mkfs.ext4 /var/tmp/fs1\n# mount -o nosync -t ext4 /var/tmp/fs1 /var/tmp/d1\n# dd if=/dev/zero of=/var/tmp/d1/fs2 bs=4096 count=1 seek=32767\n# mkfs.ext4 /var/tmp/d1/fs2\n# mount -o nosync -t ext4 /var/tmp/d1/fs2 /var/tmp/d2\n# dd if=/dev/zero of=/var/tmp/d2/test bs=4096 count=24576\n  ...will normally succeed...\n# sync\n  ...fails with I/O error!...\n```\n\n* If the filesystem is mounted `sync`, all writes to it are\n  immediately committed to the backing store, and the `dd` operation\n  above fails as soon as it fills up `/var/tmp/d1`.  However,\n  performance is drastically slowed, particularly with small writes;\n  with 1K writes, I observed performance degradation in some cases\n  exceeding three orders of magnitude.\n\n  I performed a test comparing writing 64 MB to a base (partitioned)\n  filesystem, to a loop filesystem without `sync`, and a loop\n  filesystem with `sync`.  Total I/O was sufficient to run for at least\n  5 seconds in each case.  All filesystems involved were XFS.  Loop\n  filesystems were 128 MB and dense.  Times are in seconds.  The\n  erratic behavior (e. g. the 65536 case) was involved was observed\n  repeatedly, although the exact amount of time and which I/O sizes\n  were affected varied.  The underlying device was an HP EX920 1TB\n  NVMe SSD.\n\n| I/O Size | Partition | Loop w/sync | Loop w/o sync |\n| 1024 | 0.104 | 0.120 | 140.390 |\n| 4096 | 0.045 | 0.077 | 21.850 |\n| 16384 | 0.045 | 0.067 | 5.550 |\n| 65536 | 0.044 | 0.061 | 20.440 |\n| 262144 | 0.043 | 0.087 | 0.545 |\n| 1048576 | 0.043 | 0.055 | 7.490 |\n| 4194304 | 0.043 | 0.053 | 0.587 |\n\n  The only potentially viable combination in my view would be a dense\n  loop filesystem without sync, but that would render any thin\n  provisioning impossible.\n\n## Infrastructure Needed [optional]\n\n* Decision: who is responsible for quota management of all volume\n  types (and especially ephemeral volumes of all types).  At present,\n  emptydir volumes are managed by the kubelet and logdirs and writable\n  layers by either the kubelet or the runtime, depending upon the\n  choice of runtime.  Beyond the specific proposal that the runtime\n  should manage quotas for volumes it creates, there are broader\n  issues that I request assistance from the SIG in addressing.\n\n* Location of the quota code.  If the quotas for different volume\n  types are to be managed by different components, each such component\n  needs access to the quota code.  The code is substantial and should\n  not be copied; it would more appropriately be vendored.\n\n## References\n\n### Bugs Opened Against Filesystem Quotas\n\nThe following is a list of known security issues referencing\nfilesystem quotas on Linux, and other bugs referencing filesystem\nquotas in Linux since 2012.  These bugs are not necessarily in the\nquota system.\n\n#### CVE\n\n* *CVE-2012-2133* Use-after-free vulnerability in the Linux kernel\n  before 3.3.6, when huge pages are enabled, allows local users to\n  cause a denial of service (system crash) or possibly gain privileges\n  by interacting with a hugetlbfs filesystem, as demonstrated by a\n  umount operation that triggers improper handling of quota data.\n\n  The issue is actually related to huge pages, not quotas\n  specifically.  The demonstration of the vulnerability resulted in\n  incorrect handling of quota data.\n\n* *CVE-2012-3417* The good_client function in rquotad (rquota_svc.c)\n  in Linux DiskQuota (aka quota) before 3.17 invokes the hosts_ctl\n  function the first time without a host name, which might allow\n  remote attackers to bypass TCP Wrappers rules in hosts.deny (related\n  to rpc.rquotad; remote attackers might be able to bypass TCP\n  Wrappers rules).\n\n  This issue is related to remote quota handling, which is not the use\n  case for the proposal at hand.\n\n#### Other Security Issues Without CVE\n\n* [Linux Kernel Quota Flaw Lets Local Users Exceed Quota Limits and\n  Create Large Files](https://securitytracker.com/id/1002610)\n\n  A setuid root binary inheriting file descriptors from an\n  unprivileged user process may write to the file without respecting\n  quota limits.  If this issue is still present, it would allow a\n  setuid process to exceed any enforcing limits, but does not affect\n  the quota accounting (use of quotas for monitoring).\n\n### Other Linux Quota-Related Bugs Since 2012\n\n* [ext4: report delalloc reserve as non-free in statfs mangled by\n  project quota](https://lore.kernel.org/patchwork/patch/884530/)\n\n  This bug, fixed in Feb. 2018, properly accounts for reserved but not\n  committed space in project quotas.  At this point I have not\n  determined the impact of this issue.\n\n* [XFS quota doesn't work after rebooting because of\n  crash](https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1461730)\n\n  This bug resulted in XFS quotas not working after a crash or forced\n  reboot.  Under this proposal, Kubernetes would fall back to du for\n  monitoring should a bug of this nature manifest itself again.\n\n* [quota can show incorrect filesystem\n  name](https://bugzilla.redhat.com/show_bug.cgi?id=1326527)\n\n  This issue, which will not be fixed, results in the quota command\n  possibly printing an incorrect filesystem name when used on remote\n  filesystems.  It is a display issue with the quota command, not a\n  quota bug at all, and does not result in incorrect quota information\n  being reported.  As this proposal does not utilize the quota command\n  or rely on filesystem name, or currently use quotas on remote\n  filesystems, it should not be affected by this bug.\n\nIn addition, the e2fsprogs have had numerous fixes over the years.\n"
+  },
+  {
+    "id": "f169e4562b190a989ecb448219afcaa2",
+    "title": "HugePages",
+    "authors": ["@derekwaynecarr", "@sjenning", "@PiotrProkop"],
+    "owningSig": "sig-node",
+    "participatingSigs": ["sig-scheduling"],
+    "reviewers": ["@vishnu"],
+    "approvers": ["@dawnchen"],
+    "editor": "Derek Carr",
+    "creationDate": "2019-01-29",
+    "lastUpdated": "2019-03-05",
+    "status": "implemented",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# HugePages\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories [optional]](#user-stories-optional)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n    - [Feature Gate](#feature-gate)\n    - [Node Specification](#node-specification)\n    - [Pod Specification](#pod-specification)\n    - [CRI Updates](#cri-updates)\n    - [Cgroup Enforcement](#cgroup-enforcement)\n    - [Limits and Quota](#limits-and-quota)\n    - [Scheduler changes](#scheduler-changes)\n    - [cAdvisor changes](#cadvisor-changes)\n  - [Risks and Mitigations](#risks-and-mitigations)\n    - [Huge pages as shared memory](#huge-pages-as-shared-memory)\n    - [NUMA](#numa)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n  - [Version 1.8](#version-18)\n  - [Version 1.9](#version-19)\n  - [Version 1.14](#version-114)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nA proposal to enable applications running in a Kubernetes cluster to use huge\npages.\n\nA pod may request a number of huge pages.  The `scheduler` is able to place the\npod on a node that can satisfy that request.  The `kubelet` advertises an\nallocatable number of huge pages to support scheduling decisions. A pod may\nconsume hugepages via `hugetlbfs` or `shmget`.  Huge pages are not\novercommitted.\n\n## Motivation\n\nMemory is managed in blocks known as pages.  On most systems, a page is 4Ki. 1Mi\nof memory is equal to 256 pages; 1Gi of memory is 256,000 pages, etc. CPUs have\na built-in memory management unit that manages a list of these pages in\nhardware. The Translation Lookaside Buffer (TLB) is a small hardware cache of\nvirtual-to-physical page mappings.  If the virtual address passed in a hardware\ninstruction can be found in the TLB, the mapping can be determined quickly.  If\nnot, a TLB miss occurs, and the system falls back to slower, software based\naddress translation.  This results in performance issues.  Since the size of the\nTLB is fixed, the only way to reduce the chance of a TLB miss is to increase the\npage size.\n\nA huge page is a memory page that is larger than 4Ki.  On x86_64 architectures,\nthere are two common huge page sizes: 2Mi and 1Gi.  Sizes vary on other\narchitectures, but the idea is the same.  In order to use huge pages,\napplication must write code that is aware of them.  Transparent Huge Pages (THP)\nattempts to automate the management of huge pages without application knowledge,\nbut they have limitations.  In particular, they are limited to 2Mi page sizes.\nTHP might lead to performance degradation on nodes with high memory utilization\nor fragmentation due to defragmenting efforts of THP, which can lock memory\npages. For this reason, some applications may be designed to (or recommend)\nusage of pre-allocated huge pages instead of THP.\n\nManaging memory is hard, and unfortunately, there is no one-size fits all\nsolution for all applications.\n\n### Goals\n\nThis proposal only includes pre-allocated huge pages configured on the node by\nthe administrator at boot time or by manual dynamic allocation.    \n\n### Non-Goals\n\nThis proposal defers issues relating to NUMA. It does not discuss how the\ncluster could dynamically attempt to allocate huge pages in an attempt to find a\nfit for a pod pending scheduling.  It is anticipated that operators may use a\nvariety of strategies to allocate huge pages, but we do not anticipate the\nkubelet itself doing the allocation. Allocation of huge pages ideally happens\nsoon after boot time.\n\n## Proposal\n\n### User Stories [optional]\n\nThe class of applications that benefit from huge pages typically have\n- A large memory working set\n- A sensitivity to memory access latency\n\nExample applications include:\n- database management systems (MySQL, PostgreSQL, MongoDB, Oracle, etc.)\n- Java applications can back the heap with huge pages using the\n  `-XX:+UseLargePages` and `-XX:LagePageSizeInBytes` options.\n- packet processing systems (DPDK)\n- VMs running on top of Kubernetes infrastructure (libvirt, QEMU, etc.)\n\nApplications can generally use huge pages by calling\n- `mmap()` with `MAP_ANONYMOUS | MAP_HUGETLB` and use it as anonymous memory\n- `mmap()` a file backed by `hugetlbfs`\n- `shmget()` with `SHM_HUGETLB` and use it as a shared memory segment (see Known\n  Issues).\n\n1. A pod can use huge pages with any of the prior described methods.\n1. A pod can request huge pages.\n1. A scheduler can bind pods to nodes that have available huge pages.\n1. A quota may limit usage of huge pages.\n1. A limit range may constrain min and max huge page requests.\n\n### Implementation Details/Notes/Constraints [optional]\n\n#### Feature Gate\n\nThe proposal introduces huge pages as an Alpha feature.\n\nIt must be enabled via the `--feature-gates=HugePages=true` flag on pertinent\ncomponents pending graduation to Beta.\n\n#### Node Specification\n\nHuge pages cannot be overcommitted on a node.\n\nA system may support multiple huge page sizes. For each supported huge page\nsize, the node will advertise a resource of the form `hugepages-\u003chugepagesize\u003e`.\nOn Linux, supported huge page sizes are determined by parsing the\n`/sys/kernel/mm/hugepages/hugepages-{size}kB` directory on the host. Kubernetes\nwill expose a `hugepages-\u003chugepagesize\u003e` resource using binary notation form.\nIt will convert `\u003chugepagesize\u003e` into the most compact binary notation using\ninteger values.  For example, if a node supports `hugepages-2048kB`, a resource\n`hugepages-2Mi` will be shown in node capacity and allocatable values.\nOperators may set aside pre-allocated huge pages that are not available for user\npods similar to normal memory via the `--system-reserved` flag.\n\nThere are a variety of huge page sizes supported across different hardware\narchitectures.  It is preferred to have a resource per size in order to better\nsupport quota.  For example, 1 huge page with size 2Mi is orders of magnitude\ndifferent than 1 huge page with size 1Gi.  We assume gigantic pages are even\nmore precious resources than huge pages.\n\nPre-allocated huge pages reduce the amount of allocatable memory on a node. The\nnode will treat pre-allocated huge pages similar to other system reservations\nand reduce the amount of `memory` it reports using the following formula:\n\n```\n[Allocatable] = [Node Capacity] - \n [Kube-Reserved] - \n [System-Reserved] - \n [Pre-Allocated-HugePages * HugePageSize] -\n [Hard-Eviction-Threshold]\n```\n\nThe following represents a machine with 10Gi of memory.  1Gi of memory has been\nreserved as 512 pre-allocated huge pages sized 2Mi.  As you can see, the\nallocatable memory has been reduced to account for the amount of huge pages\nreserved.\n\n```\napiVersion: v1\nkind: Node\nmetadata:\n  name: node1\n...\nstatus:\n  capacity:\n    memory: 10Gi\n    hugepages-2Mi: 1Gi\n  allocatable:\n    memory: 9Gi\n    hugepages-2Mi: 1Gi\n...  \n```\n\n#### Pod Specification\n\nContainers in a pod can cunsume huge pages by requesting pre-allocated\nhuge pages. In order to request huge pages, A pod spec must specify a certain\namount of huge pages using the resource `hugepages-\u003chugepagesize\u003e` in container\nobject. The quantity of huge pages must be a positive amount of memory in bytes.\nThe specified amount must align with the `\u003chugepagesize\u003e`; otherwise, the pod\nwill fail validation. For example, it would be valid to request\n`hugepages-2Mi: 4Mi`, but invalid to request `hugepages-2Mi: 3Mi`.\n\nThe request and limit for `hugepages-\u003chugepagesize\u003e` must match.  Similar to\nmemory, an application that requests `hugepages-\u003chugepagesize\u003e` resource is at\nminimum in the `Burstable` QoS class.\n\nIf multiple containers consume huge pages in the same pod, the request must be\nmade for each container. Similar to memory setting in cgroup sandbox, the sum of\nhuge page limits across the pod sets on pod cgroup sandbox, and the limit of\ncontainers also sets on container cgroup sandboxes individually.\n\nIf a pod consumes huge pages via `shmget`, it must run with a supplemental group\nthat matches `/proc/sys/vm/hugetlb_shm_group` on the node.  Configuration of\nthis group is outside the scope of this specification.\n\nA pod may consume multiple huge page sizes backed by the `hugetlbfs` in a single\npod spec. In this case it must use `medium: HugePages-\u003chugepagesize\u003e` notation\nfor all volume mounts.\n\nA pod may use `medium: HugePages` only if it requests huge pages of one\nsize.\n\nIn order to consume huge pages backed by the `hugetlbfs` filesystem inside the\nspecified container in the pod, it is helpful to understand the set of mount\noptions used with `hugetlbfs`.  For more details, see \"Using Huge Pages\" here:\nhttps://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt\n\n```\nmount -t hugetlbfs \\\n\t-o uid=\u003cvalue\u003e,gid=\u003cvalue\u003e,mode=\u003cvalue\u003e,pagesize=\u003cvalue\u003e,size=\u003cvalue\u003e,\\\n\tmin_size=\u003cvalue\u003e,nr_inodes=\u003cvalue\u003e none /mnt/huge\n```\n\nThe proposal recommends extending the existing `EmptyDirVolumeSource` to satisfy\nthis use case.  A new `medium=HugePages[-\u003chugepagesize\u003e]` options would be\nsupported.  To write into this volume, the pod must make a request for huge\npages.  The `pagesize` argument is inferred from the medium of the mount if\n`medium: HugePages-\u003chugepagesize\u003e` notation is used. For `medium: HugePages`\nnotation the `pagesize` argument is inferred from the resource request\n`hugepages-\u003chugepagesize\u003e`.\n\nThe existing `sizeLimit` option for `emptyDir` would restrict usage to the\nminimum value specified between `sizeLimit` and the sum of huge page limits of\nall containers in a pod. This keeps the behavior consistent with memory backed\n`emptyDir` volumes whose usage is ultimately constrained by the pod cgroup\nsandbox memory settings.  The `min_size` option is omitted as its not necessary.\nThe `nr_inodes` mount option is omitted at this time in the same manner it is\nomitted with `medium=Memory` when using `tmpfs`.\n\nThe following is a sample pod that is limited to 1Gi huge pages of size 2Mi and\n2Gi huge pages of size 1Gi. It can consume those pages using `shmget()` or via\n`mmap()` with the specified volume.\n\n```\napiVersion: v1\nkind: Pod\nmetadata:\n  name: example\nspec:\n  containers:\n...\n    volumeMounts:\n    - mountPath: /hugepages-2Mi\n      name: hugepage-2Mi\n    - mountPath: /hugepages-1Gi\n      name: hugepage-1Gi\n    resources:\n      requests:\n        hugepages-2Mi: 1Gi\n        hugepages-1Gi: 2Gi\n      limits:\n        hugepages-2Mi: 1Gi\n        hugepages-1Gi: 2Gi\n  volumes:\n  - name: hugepage-2Mi\n    emptyDir:\n      medium: HugePages-2Mi\n  - name: hugepage-1Gi\n    emptyDir:\n      medium: HugePages-1Gi\n```\n\nFor backwards compatibility, a pod that uses one page size should pass\nvalidation if a volume emptyDir `medium=HugePages` notation is used.\n\nThe following is an example of a pod backward compatible with the\ncurrent implementation. It uses `medium: HugePages` notation and\nrequests hugepages of one size.\n\n```\napiVersion: v1\nkind: Pod\nmetadata:\n  name: example\nspec:\n  containers:\n...\n    volumeMounts:\n    - mountPath: /hugepages\n      name: hugepage\n    resources:\n      requests:\n        hugepages-2Mi: 1Gi\n      limits:\n        hugepages-2Mi: 1Gi\n  volumes:\n  - name: hugepage\n    emptyDir:\n      medium: HugePages\n```\n\nA pod that requests more than one page size should fail validation if a volume\nemptyDir medium=HugePages is specified.\n\nThis is an example of an invalid pod that requests huge pages of two\ndifferfent sizes, but doesn't use `medium: Hugepages-\u003csize\u003e` notation:\n\n```\napiVersion: v1\nkind: Pod\nmetadata:\n  name: example\nspec:\n  containers:\n...\n    volumeMounts:\n    - mountPath: /hugepages\n      name: hugepage\n    resources:\n      requests:\n        hugepages-2Mi: 1Gi\n        hugepages-1Gi: 2Gi\n      limits:\n        hugepages-2Mi: 1Gi\n        hugepages-1Gi: 2Gi\n  volumes:\n  - name: hugepage\n    emptyDir:\n      medium: HugePages\n```\n\nA pod that requests huge pages of one size and uses another size in a\nvolume emptyDir medium should fail validation.\nThis is an example of such an invalid pod. It requests hugepages of 2Mi\nsize, but specifies 1Gi in `medium: HugePages-1Gi`:\n\n```\napiVersion: v1\nkind: Pod\nmetadata:\n  name: example\nspec:\n  containers:\n...\n    volumeMounts:\n    - mountPath: /hugepages\n      name: hugepage\n    resources:\n      requests:\n        hugepages-2Mi: 1Gi\n      limits:\n        hugepages-2Mi: 1Gi\n  volumes:\n  - name: hugepage\n    emptyDir:\n      medium: HugePages-1Gi\n```\n\nAlso, it is important to note that emptyDir usage is not required if pod\nconsumes huge pages via shmat/shmget system calls or mmap with MAP_HUGETLB.\nThis is an example of the pod that consumes 1Gi huge pages of size 2Mi and\n2Gi huge pages of size 1Gi without using emptyDir:\n\n```\napiVersion: v1\nkind: Pod\nmetadata:\n  name: example\nspec:\n  containers:\n...\n    resources:\n      requests:\n        hugepages-2Mi: 1Gi\n        hugepages-1Gi: 2Gi\n      limits:\n        hugepages-2Mi: 1Gi\n        hugepages-1Gi: 2Gi\n```\n\nThe following is an example of the pod that requests multiple sizes of\nhuge pages for multiple containers. It requests 1Gi huge pages of size 1Gi and\n2Mi for the container1 and 1Gi huge pages of size 2Mi for the container2 with\nemptyDir backing. Note that `hugetlbfs` offers `size` mount option to specify\nthe maximum amount of memory for the mount, but huge pages medium does not use\nthe option to set limits so that the huge pages usage of containers will be\ncontrolled by container cgroup sandboxes individually:\n```\napiVersion: v1\nkind: Pod\nmetadata:\n  name: example\nspec:\n  containers:\n  - name: container1\n    volumeMounts:\n    - mountPath: /hugepage-2Mi\n      name: hugepage-2Mi\n    - mountPath: /hugepage-1Gi\n      name: hugepage-1Gi\n    resources:\n      requests:\n        hugepages-2Mi: 1Gi\n        hugepages-1Gi: 1Gi\n      limits:\n        hugepages-2Mi: 1Gi\n        hugepages-1Gi: 1Gi\n  - name: container2\n    volumeMounts:\n    - mountPath: /hugepage-2Mi\n      name: hugepage-2Mi\n    resources:\n      requests:\n        hugepages-2Mi: 1Gi\n      limits:\n        hugepages-2Mi: 1Gi\n  volumes:\n  - name: hugepage-2Mi\n    emptyDir:\n      medium: HugePages-2Mi\n  - name: hugepage-1Gi\n    emptyDir:\n      medium: HugePages-1Gi\n```\n\n#### CRI Updates\n\nThe `LinuxContainerResources` message should be extended to support specifying\nhuge page limits per size.  The specification for huge pages should align with\nopencontainers/runtime-spec.\nsee:\nhttps://github.com/opencontainers/runtime-spec/blob/master/config-linux.md#huge-page-limits\n\nThe runtime-spec provides the object `hugepageLimits` as an array of objects to\nrepresent `hugetlb` controller. `hugepageLimits` allows specifying `limit` per\n`pageSize`. The following is an example of the `hugepageLimits` object,\nwhich has limits per 2MB and 64KB page size:\n\n```\n    \"hugepageLimits\": [\n        {\n            \"pageSize\": \"2MB\",\n            \"limit\": 209715200\n        },\n        {\n            \"pageSize\": \"64KB\",\n            \"limit\": 1000000\n        }\n   ]\n```\n\nThe `LinuxContainerResources` message can be extended to specify multiple sizes\nto align with the runtime-spec in this way:\n```\nmessage LinuxContainerResources {\n    ...\n    string cpuset_mems = 7;\n    // List of HugepageLimits to limit the HugeTLB usage of container per page size. Default: nil (not specified).\n    repeated HugepageLimit hugepage_limits = 8;\n}\n\n// HugepageLimit corresponds to the file`hugetlb.\u003chugepagesize\u003e.limit_in_byte` in container level cgroup.\n// For example, `PageSize=1GB`, `Limit=1073741824` means setting `1073741824` bytes to hugetlb.1GB.limit_in_bytes.\nmessage HugepageLimit {\n    // The value of PageSize has the format \u003csize\u003e\u003cunit-prefix\u003eB (2MB, 1GB),\n    // and must match the \u003chugepagesize\u003e of the corresponding control file found in `hugetlb.\u003chugepagesize\u003e.limit_in_bytes`.\n    // The values of \u003cunit-prefix\u003e are intended to be parsed using base 1024(\"1KB\" = 1024, \"1MB\" = 1048576, etc).\n    string page_size = 1;\n    // limit in bytes of hugepagesize HugeTLB usage.\n    uint64 limit = 2;\n}\n```\n\n#### Cgroup Enforcement\n\nTo use this feature, the `--cgroups-per-qos` must be enabled.  In addition, the\n`hugetlb` cgroup must be mounted.\n\nThe `kubepods` cgroup is bounded by the `Allocatable` value.\n\nThe QoS level cgroups are left unbounded across all huge page pool sizes.\n\nThe pod level cgroup sandbox is configured as follows, where `hugepagesize` is\nthe system supported huge page size(s).  If no request is made for huge pages of\na particular size, the limit is set to 0 for all supported types on the node.\n\n```\npod\u003cUID\u003e/hugetlb.\u003chugepagesize\u003e.limit_in_bytes = sum(pod.spec.containers.resources.limits[hugepages-\u003chugepagesize\u003e])\n```\n\nIf the container runtime supports specification of huge page limits, the\ncontainer cgroup sandbox will be configured with the specified limit.\n\nThe `kubelet` will ensure the `hugetlb` has no usage charged to the pod level\ncgroup sandbox prior to deleting the pod to ensure all resources are reclaimed.\n\n#### Limits and Quota\n\nThe `ResourceQuota` resource will be extended to support accounting for\n`hugepages-\u003chugepagesize\u003e` similar to `cpu` and `memory`.  The `LimitRange`\nresource will be extended to define min and max constraints for `hugepages`\nsimilar to `cpu` and `memory`.\n\n#### Scheduler changes\n\nThe scheduler will need to ensure any huge page request defined in the pod spec\ncan be fulfilled by a candidate node.\n\n#### cAdvisor changes\n\ncAdvisor will need to be modified to return the number of pre-allocated huge\npages per page size on the node.  It will be used to determine capacity and\ncalculate allocatable values on the node.\n\n### Risks and Mitigations\n\n#### Huge pages as shared memory\n\nFor the Java use case, the JVM maps the huge pages as a shared memory segment\nand memlocks them to prevent the system from moving or swapping them out.\n\nThere are several issues here:\n- The user running the Java app must be a member of the gid set in the\n  `vm.huge_tlb_shm_group` sysctl\n- sysctl `kernel.shmmax` must allow the size of the shared memory segment\n- The user's memlock ulimits must allow the size of the shared memory segment\n- `vm.huge_tlb_shm_group` is not namespaced.\n\n#### NUMA\n\nNUMA is complicated.  To support NUMA, the node must support cpu pinning,\ndevices, and memory locality.  Extending that requirement to huge pages is not\nmuch different.  It is anticipated that the `kubelet` will provide future NUMA\nlocality guarantees as a feature of QoS.  In particular, pods in the\n`Guaranteed` QoS class are expected to have NUMA locality preferences.\n\n## Graduation Criteria\n\n- Reports of successful usage of the feature for isolating huge page resources.\n- E2E testing validating its usage.\n-- https://k8s-testgrid.appspot.com/sig-node-kubelet#node-kubelet-serial\u0026include-filter-by-regex=Feature%3AHugePages\n\n## Implementation History\n\n### Version 1.8\n\nInitial alpha support for huge pages usage by pods.\n\n### Version 1.9\n\nBeta support for huge pages\n\n### Version 1.14\n\nGA support for huge pages proposed based on feedback from user community\nusing the feature without issue.\n"
+  },
+  {
+    "id": "f0bda42831f39fdff479a91f9609cd32",
+    "title": "Pid Limiting",
+    "authors": ["@derekwaynecarr", "@dims"],
+    "owningSig": "sig-node",
+    "participatingSigs": null,
+    "reviewers": ["@dashpole"],
+    "approvers": ["@dashpole", "@dchen1107"],
+    "editor": "Derek Carr",
+    "creationDate": "2019-01-29",
+    "lastUpdated": "2019-03-05",
+    "status": "implemented",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Pid Limiting\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories [optional]](#user-stories-optional)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n    - [Pod to Pod Isolation](#pod-to-pod-isolation)\n    - [Node to Pod Isolation](#node-to-pod-isolation)\n    - [Cgroup Enforcement](#cgroup-enforcement)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n  - [Pod to Pod pid isolation](#pod-to-pod-pid-isolation)\n  - [Node to Pod pid isolation](#node-to-pod-pid-isolation)\n- [Implementation History](#implementation-history)\n  - [Version 1.10](#version-110)\n  - [Version 1.14](#version-114)\n  - [Version 1.15](#version-115)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nA proposal to enable isolation of pid resources.  It proposes a mechanism to\nenable pod-to-pod PID isolation as well as node-to-pod PID isolation.\n\n## Motivation\n\nPids are a fundamental resource on Linux hosts.  It is trivial to hit the task\nlimit without hitting any other resource limits and cause instability to a host\nmachine.\n\nAdministrators require mechanisms to ensure that user pods cannot induce pid\nexhaustion that prevents host daemons (runtime, kubelet, etc) from running.  In\naddition, it is important to ensure that pids are limited among pods in order to\nensure they have limited impact to other workloads on the node.\n\n### Goals\n\nThis proposal aims to the following:\n- enable administrator control to provide pod-to-pod pid isolation\n- enable administrator control to provide node-to-pod pid isolation\n\n### Non-Goals\n\nThis proposal defers the following:\n- ability for a user to request additional number of pid resources per pod\n\nIt is anticipated we will support that via a policy knob that could be\nrestricted and/or defaulted via PodSecurityPolicy or LimitRange.  We anticipate\ntracking this work under a separate feature gate `GranularPidLimitsPerPod`.  Any\ndefaulting applied to pods today would only be used if the pod had no local pod\npid limiting policy in future dates.\n\n## Proposal\n\n### User Stories [optional]\n\n1. Administrator can default the number of pids per pod to provide pod-to-pod\n   isolation.\n1. Administrator can reserve a number of allocatable pids to user pods via node\n   allocatable.\n\n### Implementation Details/Notes/Constraints [optional]\n\n#### Pod to Pod Isolation\n\nTo enable pid isolation among pods, the `SupportPodPidsLimit` feature gate is\ndefined.\n\nIf enabled, the kubelet argument for `pod-max-pids` will write out the\nconfigured pid limit to the pod level cgroup to the value specified on Linux\nhosts.  If -1, the kubelet will default to the node allocatable pid capacity.\n\n#### Node to Pod Isolation\n\nTo enable pid isolation from node to pods, the `SupportNodePidsLimit` feature\ngate is proposed.  If enabled, pid reservations may be supported at the node\nallocatable and eviction manager subsystem configurations.\n\nNode allocatable is a well-established feature concept in the kubelet that\nallows isolation of user pod resources from host daemons at the `kubepods`\ncgroup level that parents all end-user pods.\n\nThe kubelet will be updated to support reservation of pids so the effective pid\nlimit is enabled as follows:\n\n```\n[Allocatable] = [Node Capacity] - \n [Kube-Reserved] - \n [System-Reserved] - \n [Hard-Eviction-Threshold]\n```\n\n#### Cgroup Enforcement\n\nTo use this feature, the `--cgroups-per-qos` must be enabled.  In addition, the\n`pids` cgroup must be mounted.\n\nThe `kubepods` cgroup is bounded by the `Allocatable` value.\n\nThe QoS level cgroups are left unbounded across all pid pool sizes.\n\nThe pod level cgroup sandbox is configured as follows:\n\n1. the pod-max-pids value if positive and is specified on kubelet config\n1. the local pod pid limiting policy (future)\n1. unbounded (so it is restricted by the `Allocatable` value at `kubepods`)\n\n### Risks and Mitigations\n\nNone\n\n## Graduation Criteria\n\n### Pod to Pod pid isolation\n\nThe following criteria applies to `SupportPodPidsLimit` feature gate:\n\nAlpha\n- basic support integrated in kubelet\n\nBeta\n- ensure proper node e2e test coverage is integrated verifying cgroup settings\n- see testing:\nhttps://github.com/kubernetes/kubernetes/blob/master/test/e2e_node/pids_test.go\nhttps://k8s-testgrid.appspot.com/sig-node-kubelet#node-kubelet-serial\u0026include-filter-by-regex=Feature%3ASupportPodPidsLimit\n\nGA\n- assuming no negative user feedback based on production experience, promote\n  after 2 releases in beta.\n\n### Node to Pod pid isolation\n\nAdding support for pid limiting at the Node Allocatable level \n\nThe following criteria applies to `SupportNodePidsLimit`:\n\nAlpha\n- basic support integrated via eviction manager and/or node allocatable level\n\nBeta\n- ensure proper node e2e testing coverage to ensure a pod is unable to fork-bomb\n  a node even when `pod-max-pids` is unbounded.\n- see testing:\nhttps://github.com/kubernetes/kubernetes/pull/73651/files#diff-7681b587a8fd514b312fa29c3acc669e\n\n\nGA\n- assuming no negative user feedback, promote after 1 release at beta.\n\n## Implementation History\n\n### Version 1.10\n\n`SupportPodPidsLimit` implemented at Alpha.\n\n### Version 1.14\n\n- Implement `SupportNodePidsLimit` as Alpha.\n- Graduate `SupportPodPidsLimit` to Beta by adding node e2e test coverage for\n  pid cgroup isolation, ensure PidPressure works as intended.\n  \n### Version 1.15\n\n- Graduate `SupportNodePidsLimit` to beta by adding node e2e test\n  coverage for node cgroup isoation.\n"
+  },
+  {
+    "id": "5809e0c9225e0ac6a3a6a6b1e23832e4",
+    "title": "Promote Node Operating System \u0026 Architecture labels to GA",
+    "authors": ["@yujuhong"],
+    "owningSig": "sig-node",
+    "participatingSigs": ["sig-node"],
+    "reviewers": ["@liggitt"],
+    "approvers": ["@dchen1107"],
+    "editor": "Yu-Ju Hong",
+    "creationDate": "2019-01-30",
+    "lastUpdated": "2019-01-30",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Promote Node Operating System \u0026 Architecture labels to GA\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThe kubelet has been been labeling the Node object with operating system (OS)\nand architecture (arch) labels since Kubernetes 1.3. This proposal aims to\npromote these labels to GA and ensure a smooth transition with backward\ncompatibility.\n\nBelow lists the labels to promote:\n```\nbeta.kubernetes.io/os\nbeta.kubernetes.io/arch\n```\n\n## Motivation\n\nThe labels have existed over two years and are widely in various places.\nPromoting these labels to reflect their stable status.\n\n### Goals\n\nPromote the OS and arch labels to GA.\n\n### Non-Goals\n\nPromoting any other label that is not listed in the Goals sections.\n\n## Proposal\n\nA new set of GA labels will be added in 1.14:\n```\nkubernetes.io/os\nkubernetes.io/arch\n```\nBased on the [deprecation\npolicy](https://kubernetes.io/docs/reference/using-api/deprecation-policy/#deprecating-a-feature-or-behavior),\nkubelet will continue reporting beta labels until 1.18.\n\nIn *v1.14*,\n- announce the deprecation of the beta OS and GA labels, and the timeframe for removal (v1.18)\n- update documentation and examples to the GA labels (noting which version the GA labels were added)\n\nStarting in *v1.14*,\n- kubelets will report *both* the beta and the GA labels. \n- the node controller will reconcile a mismatch between a present or missing\n  GA label in favor of the beta label to ensure uniform labeling.\n  - Pre-1.14 kubelets will only report beta labels.\n  - 1.14+ kubelets report both the beta and GA labels.\n  - If a GA label does not exist for a node, add one to match the beta label.\n  - If the beta label does not match the GA label, modify the GA label to match\n    the beta label.\n\nIn *v1.15*,\n- update in-org use in manifests targeting 1.15+ kubernetes versions to the GA labels\n\nStarting in *v1.18*, \n- kubelet will stop reporting the beta labels.\n- the node controller will switch to reconciling a mismatch between a present\n  beta label and a present GA label in favor of the GA label, since pre-1.14\n  kubelets are not supported against a 1.18+ control plane.\n  - All nodes of supported versions should report GA labels.\n  - If a beta label does not exist for a node, do nothing.\n  - If the beta label exists but does not match the GA label, modfiy the beta\n    label to match the GA label.\n\n### Risks and Mitigations\n\nThe risk is that nodes may have inconsistent labels after upgrades \u0026\ndowngrades. This is addressed by instructing the node controller to reconcile\nand ensure uniform labeling.\n\n## Graduation Criteria\n\n- GA labels are consistently available, regardless of kubelet version\n- Examples and documentation direct users to use the new labels\n- In-org manifests use the new labels\n"
+  },
+  {
+    "id": "9f784e8a59c0693548a3acf8bd26287b",
+    "title": "RunAsGroup support in PodSpec and PodSecurityPolicy",
+    "authors": ["@krmayankk"],
+    "owningSig": "sig-node",
+    "participatingSigs": ["sig-auth"],
+    "reviewers": ["@tallclair", "@mrunalp"],
+    "approvers": ["@liggitt", "@derekwaynecarr"],
+    "editor": "TBD",
+    "creationDate": "2017-06-21",
+    "lastUpdated": "2019-02-14",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# RunAsGroup support in PodSpec and PodSecurityPolicy\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Abstract](#abstract)\n- [Motivation](#motivation)\n  - [What is the significance of Primary Group Id?](#what-is-the-significance-of-primary-group-id)\n- [Goals](#goals)\n- [Use Cases](#use-cases)\n  - [Use Case 1:](#use-case-1)\n  - [Use Case 2:](#use-case-2)\n- [Design](#design)\n  - [Model](#model)\n    - [SecurityContext](#securitycontext)\n    - [PodSecurityContext](#podsecuritycontext)\n    - [PodSecurityPolicy](#podsecuritypolicy)\n- [Behavior](#behavior)\n  - [Note About RunAsNonRoot field](#note-about-runasnonroot-field)\n- [Summary of Changes needed](#summary-of-changes-needed)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Abstract\nAs a Kubernetes User, we should be able to specify both user id and group id for the containers running \ninside a pod on a per Container basis, similar to how docker allows that using docker run options `-u`, \n```\n-u, --user=\"\" Username or UID (format: \u003cname|uid\u003e[:\u003cgroup|gid\u003e]) format\n```\n\nPodSecurityContext allows Kubernetes users to specify RunAsUser which can be overridden by RunAsUser\nin SecurityContext on a per Container basis. There is no equivalent field for specifying the primary\nGroup of the running container.\n\n## Motivation\nEnterprise Kubernetes users want to run containers as non root. This means running containers with a \nnon zero user id and non zero primary group id. This gives Enterprises, confidence that their customer code\nis running with least privilege and if it escapes the container boundary, will still cause least harm\nby decreasing the attack surface.\n\n### What is the significance of Primary Group Id?\nPrimary Group Id is the group id used when creating files and directories. It is also the default group \nassociated with a user, when he/she logins. All groups are defined in `/etc/group` file and are created\nwith the `groupadd` command. A Process/Container runs with uid/primary gid of the calling user. If no\nprimary group is specified for a user, 0(root) group is assumed. This means, any files/directories created\nby a process running as user with no primary group associated with it, will be owned by group id 0(root).\n\n## Goals\n\n1. Provide the ability to specify the Primary Group id for a container inside a Pod\n2. Bring launching of containers using Kubernetes at par with Dockers by supporting the same features.\n\n\n## Use Cases\n\n### Use Case 1:\nAs a Kubernetes User, I should be able to control both user id and primary group id of containers \nlaunched using Kubernetes at runtime, so that i can run the container as non root with least possible\nprivilege.\n\n### Use Case 2:\nAs a Kubernetes User, I should be able to control both user id and primary group id of containers \nlaunched using Kubernetes at runtime, so that i can override the user id and primary group id specified\nin the Dockerfile of the container image, without having to create a new Docker image.\n\n## Design\n\n### Model\n\nIntroduce a new API field in SecurityContext and PodSecurityContext called `RunAsGroup`.\n\n#### SecurityContext\n\n```\n// SecurityContext holds security configuration that will be applied to a container.\n// Some fields are present in both SecurityContext and PodSecurityContext.  When both\n// are set, the values in SecurityContext take precedence.\ntype SecurityContext struct {\n     //Other fields not shown for brevity\n    ..... \n\n     // The UID to run the entrypoint of the container process.\n     // Defaults to user specified in image metadata if unspecified.\n     // May also be set in PodSecurityContext.  If set in both SecurityContext and\n     // PodSecurityContext, the value specified in SecurityContext takes precedence.\n     // +optional\n     RunAsUser *int64\n     // The GID to run the entrypoint of the container process.\n     // Defaults to group specified in image metadata if unspecified.\n     // May also be set in PodSecurityContext.  If set in both SecurityContext and\n     // PodSecurityContext, the value specified in SecurityContext takes precedence.\n     // +optional\n     RunAsGroup *int64\n     // Indicates that the container must run as a non-root user.\n     // If true, the Kubelet will validate the image at runtime to ensure that it\n     // does not run as UID 0 (root) and fail to start the container if it does.\n     // If unset or false, no such validation will be performed.\n     // May also be set in SecurityContext.  If set in both SecurityContext and\n     // PodSecurityContext, the value specified in SecurityContext takes precedence.\n     // +optional\n     RunAsNonRoot *bool\n\n    .....\n }\n```\n\n#### PodSecurityContext \n\n```\ntype PodSecurityContext struct {\n     //Other fields not shown for brevity\n    ..... \n\n     // The UID to run the entrypoint of the container process.\n     // Defaults to user specified in image metadata if unspecified.\n     // May also be set in SecurityContext.  If set in both SecurityContext and\n     // PodSecurityContext, the value specified in SecurityContext takes precedence\n     // for that container.\n     // +optional\n     RunAsUser *int64\n     // The GID to run the entrypoint of the container process.\n     // Defaults to group specified in image metadata if unspecified.\n     // May also be set in PodSecurityContext.  If set in both SecurityContext and\n     // PodSecurityContext, the value specified in SecurityContext takes precedence.\n     // +optional\n     RunAsGroup *int64\n     // Indicates that the container must run as a non-root user.\n     // If true, the Kubelet will validate the image at runtime to ensure that it\n     // does not run as UID 0 (root) and fail to start the container if it does.\n     // If unset or false, no such validation will be performed.\n     // May also be set in SecurityContext.  If set in both SecurityContext and\n     // PodSecurityContext, the value specified in SecurityContext takes precedence.\n     // +optional\n     RunAsNonRoot *bool\n\n    .....\n }\n```\n\n#### PodSecurityPolicy\n\nPodSecurityPolicy defines strategies or conditions that a pod must run with in order to be accepted\ninto the system. Two of the relevant strategies are RunAsUser and SupplementalGroups. We introduce \na new strategy called RunAsGroup which will support the following options:\n- MustRunAs\n- RunAsAny\n\n```\n// PodSecurityPolicySpec defines the policy enforced.\n type PodSecurityPolicySpec struct {\n     //Other fields not shown for brevity\n    ..... \n  // RunAsUser is the strategy that will dictate the allowable RunAsUser values that may be set.\n  RunAsUser RunAsUserStrategyOptions\n  // SupplementalGroups is the strategy that will dictate what supplemental groups are used by the SecurityContext.\n  SupplementalGroups SupplementalGroupsStrategyOptions\n\n\n  // RunAsGroup is the strategy that will dictate the allowable RunAsGroup values that may be set.\n  RunAsGroup RunAsGroupStrategyOptions\n   .....\n}\n\n// RunAsGroupStrategyOptions defines the strategy type and any options used to create the strategy.\n type RunAsUserStrategyOptions struct {\n     // Rule is the strategy that will dictate the allowable RunAsGroup values that may be set.\n     Rule RunAsGroupStrategy\n     // Ranges are the allowed ranges of gids that may be used.\n     // +optional\n     Ranges []GroupIDRange\n }\n\n// RunAsGroupStrategy denotes strategy types for generating RunAsGroup values for a\n // SecurityContext.\n type RunAsGroupStrategy string\n \n const (\n     // container must run as a particular gid.\n     RunAsGroupStrategyMustRunAs RunAsGroupStrategy = \"MustRunAs\"\n     // container may make requests for any gid.\n     RunAsGroupStrategyRunAsAny RunAsGroupStrategy = \"RunAsAny\"\n )\n```\n\n## Behavior\n\nFollowing points should be noted:\n\n- `FSGroup` and `SupplementalGroups` will continue to have their old meanings and would be untouched.  \n- The `RunAsGroup` In the SecurityContext will override the `RunAsGroup` in the PodSecurityContext.\n- If both `RunAsUser` and `RunAsGroup` are NOT provided, the USER field in Dockerfile is used\n- If both `RunAsUser` and `RunAsGroup` are specified, that is passed directly as User.\n- If only one of `RunAsUser` or `RunAsGroup` is specified, the remaining value is decided by the Runtime,\n  where the Runtime behavior is to make it run with uid or gid as 0.\n\nBasically, we guarantee to set the values provided by user, and the runtime dictates the rest.\n\nHere is an example of what gets passed to docker User\n- runAsUser set to 9999, runAsGroup set to 9999 -\u003e Config.User set to 9999:9999\n- runAsUser set to 9999, runAsGroup unset -\u003e Config.User set to 9999 -\u003e docker runs you with 9999:0\n- runAsUser unset, runAsGroup set to 9999 -\u003e Config.User set to :9999 -\u003e docker runs you with 0:9999 \n- runAsUser unset, runAsGroup unset -\u003e Config.User set to whatever is present in Dockerfile\nThis is to keep the behavior backward compatible and as expected.\n\n### Note About RunAsNonRoot field\n\nNote that this change does not introduce an equivalent field called runAsNonRootGroup in both SecurityContext\nand PodSecurityContext. There was ongoing discussion about this field at PR [#62216](https://github.com/kubernetes/kubernetes/pull/62217)\nThe summary of this discussion seems as follows:-\n- Use PSP MustRunAs Group strategy to guarantee that Pod never runs with 0 as Primary Group ID.\n- Using the PSP MustRunAs Group strategy forces Pod to always specify a RunAsGroup\n- RunAsGroup field when specified in PodSpec, will always override USER field in Dockerfile\n\nThere are other potentially unresolved discussions in that PR which need a followup.\n\n## Summary of Changes needed\n- https://github.com/kubernetes/kubernetes/pull/52077\n- https://github.com/kubernetes/kubernetes/pull/67802\n- https://github.com/kubernetes/kubernetes/pull/61030\n- https://github.com/kubernetes/kubernetes/pull/72230\n- https://github.com/kubernetes/kubernetes/pull/70465\n- https://github.com/kubernetes/website/pull/12297\n- https://github.com/kubernetes/kubernetes/pull/73007\n\n\n## Graduation Criteria\n\n- Publish Test Results from Master Branch of Cri-o To http://prow.k8s.io [#72253](https://github.com/kubernetes/kubernetes/issues/72253)\n- Containerd and CRI-O tests included in k/k CI [#72287](https://github.com/kubernetes/kubernetes/issues/72287)\n- Make CRI tests failures as release informing\n\n## Implementation History\n- Proposal merged on 9-18-2017\n- Implementation merged as Alpha on 3-1-2018 and Release in 1.10\n- Implementation for Containerd merged on 3-30-2018 \n- Implementation for CRI-O merged on 6-8-2018\n- Implemented RunAsGroup PodSecurityPolicy Strategy on 10-12-2018\n- Planned Beta in v1.14\n"
+  },
+  {
+    "id": "d27de50641da7a19b65c9fd743b93266",
+    "title": "Ephemeral Containers",
+    "authors": ["@verb"],
+    "owningSig": "sig-node",
+    "participatingSigs": ["sig-auth", "sig-node"],
+    "reviewers": ["@yujuhong"],
+    "approvers": ["@dchen1107", "@liggitt"],
+    "editor": "TBD",
+    "creationDate": "2019-02-12",
+    "lastUpdated": "2019-10-02",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Ephemeral Containers\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Development](#development)\n  - [Operations and Support](#operations-and-support)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Kubernetes API Changes](#kubernetes-api-changes)\n    - [Pod Changes](#pod-changes)\n      - [Alternative Considered: Omitting TargetContainerName](#alternative-considered-omitting-targetcontainername)\n    - [Updating a Pod](#updating-a-pod)\n  - [Container Runtime Interface (CRI) changes](#container-runtime-interface-cri-changes)\n  - [Creating Ephemeral Containers](#creating-ephemeral-containers)\n  - [Restarting and Reattaching Ephemeral Containers](#restarting-and-reattaching-ephemeral-containers)\n  - [Killing Ephemeral Containers](#killing-ephemeral-containers)\n  - [User Stories](#user-stories)\n    - [Operations](#operations)\n    - [Debugging](#debugging)\n    - [Automation](#automation)\n    - [Technical Support](#technical-support)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n  - [Risks and Mitigations](#risks-and-mitigations)\n    - [Security Considerations](#security-considerations)\n    - [Requiring a Subresource](#requiring-a-subresource)\n    - [Creative New Uses of Ephemeral Containers](#creative-new-uses-of-ephemeral-containers)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n  - [Container Spec in PodStatus](#container-spec-in-podstatus)\n  - [Extend the Existing Exec API (\u0026quot;exec++\u0026quot;)](#extend-the-existing-exec-api-exec)\n  - [Ephemeral Container Controller](#ephemeral-container-controller)\n  - [Mutable Pod Spec Containers](#mutable-pod-spec-containers)\n  - [Image Exec](#image-exec)\n  - [Attaching Container Type Volume](#attaching-container-type-volume)\n  - [Using docker cp and exec](#using-docker-cp-and-exec)\n  - [Inactive container](#inactive-container)\n  - [Implicit Empty Volume](#implicit-empty-volume)\n  - [Standalone Pod in Shared Namespace (\u0026quot;Debug Pod\u0026quot;)](#standalone-pod-in-shared-namespace-debug-pod)\n  - [Exec from Node](#exec-from-node)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e., [kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be updated for the enhancement to be released.\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [x] KEP approvers have set the KEP status to `implementable`\n- [x] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nThis proposal adds to Kubernetes a mechanism to run a container with a\ntemporary duration that executes within namespaces of an existing pod.\nEphemeral Containers are initiated by a user and intended to observe the state\nof other pods and containers for troubleshooting and debugging purposes.\n\nEphemeral Containers unlock the possibility for a new command, `kubectl debug`,\nwhich parallels the existing, `kubectl exec`.  Whereas `kubectl exec` runs a\n_process_ in a _container_, `kubectl debug` could run a _container_ in a _pod_.\n\nFor example, the following command would attach to a newly created container in\na pod:\n\n```\nkubectl debug -c debug-shell --image=debian target-pod -- bash\n```\n\n## Motivation\n\n### Development\n\nMany developers of native Kubernetes applications wish to treat Kubernetes as an\nexecution platform for custom binaries produced by a build system. These users\ncan forgo the scripted OS install of traditional Dockerfiles and instead `COPY`\nthe output of their build system into a container image built `FROM scratch` or\na\n[distroless container image](https://github.com/GoogleCloudPlatform/distroless).\nThis confers several advantages:\n\n1.  **Minimal images** lower operational burden and reduce attack vectors.\n1.  **Immutable images** improve correctness and reliability.\n1.  **Smaller image size** reduces resource usage and speeds deployments.\n\nThe disadvantage of using containers built `FROM scratch` is the lack of system\nbinaries provided by a Linux distro image makes it difficult to\ntroubleshoot running containers. Kubernetes should enable one to troubleshoot\npods regardless of the contents of the container images.\n\nOn Windows, the minimal [Nano Server](https://hub.docker.com/_/microsoft-windows-nanoserver)\nimage is the smallest available, which still retains the `cmd` shell and some\ncommon tools such as `curl.exe`. This makes downloading debugger scripts and\ntools feasible today during a `kubectl exec` session without the need for a\nseparate ephemeral container. Windows cannot build containers `FROM scratch`.\n\n### Operations and Support\n\nAs Kubernetes gains in popularity, it's becoming the case that a person\ntroubleshooting an application is not necessarily the person who built it.\nOperations staff and Support organizations want the ability to attach a \"known\ngood\" or automated debugging environment to a pod.\n\n### Goals\n\nIn order to support the debugging use case, Ephemeral Containers must:\n\n*   allow access to namespaces and the file systems of individual containers\n*   fetch container images at run time rather than at the time of pod or image\n    creation\n*   respect admission controllers and audit logging\n*   be discoverable via the API\n*   support arbitrary runtimes via the CRI (possibly with reduced feature set)\n*   require no administrative access to the node\n*   have no _inherent_ side effects to the running container image\n*   define a v1.Container available for inspection by admission controllers\n\n### Non-Goals\n\nEven though this proposal makes reference to a `kubectl debug`, implementation\nof this user-level command is out of scope. This KEP focuses on the API and\nkubelet changes required to enable such a debugging experience.\n\nA method for debugging using Ephemeral Containers should be proposed in a\nseparate KEP or implemented via `kubectl` plugins.\n\nPods running on Windows Server 2019 will not have feature parity and support\nall the user stories described here. Only the network troubleshooting user\nstory detailed under [Operations](#operations) would be feasible.\n\n## Proposal\n\n### Kubernetes API Changes\n\nEphemeral Containers are implemented in the Core API to avoid new dependencies\nin the kubelet.  The API doesn't require an Ephemeral Container to be used for\ndebugging. It's intended as a general purpose construct for running a\nshort-lived container in a pod.\n\n#### Pod Changes\n\nEphemeral Containers are represented in `PodSpec` and `PodStatus`:\n\n```\ntype PodSpec struct {\n\t...\n\t// List of user-initiated ephemeral containers to run in this pod.\n\t// This field is alpha-level and is only honored by servers that enable the EphemeralContainers feature.\n\t// +optional\n\t// +patchMergeKey=name\n\t// +patchStrategy=merge\n\tEphemeralContainers []EphemeralContainer `json:\"ephemeralContainers,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"name\" protobuf:\"bytes,34,rep,name=ephemeralContainers\"`\n}\n\ntype PodStatus struct {\n\t...\n\t// Status for any Ephemeral Containers that running in this pod.\n\t// This field is alpha-level and is only honored by servers that enable the EphemeralContainers feature.\n\t// +optional\n\tEphemeralContainerStatuses []ContainerStatus `json:\"ephemeralContainerStatuses,omitempty\" protobuf:\"bytes,13,rep,name=ephemeralContainerStatuses\"`\n}\n```\n\n`EphemeralContainerStatuses` resembles the existing `ContainerStatuses` and\n`InitContainerStatuses`, but `EphemeralContainers` introduces a new type:\n\n```\n// An EphemeralContainer is a container that may be added temporarily to an existing pod for\n// user-initiated activities such as debugging. Ephemeral containers have no resource or\n// scheduling guarantees, and they will not be restarted when they exit or when a pod is\n// removed or restarted. If an ephemeral container causes a pod to exceed its resource\n// allocation, the pod may be evicted.\n// Ephemeral containers may not be added by directly updating the pod spec. They must be added\n// via the pod's ephemeralcontainers subresource, and they will appear in the pod spec\n// once added.\n// This is an alpha feature enabled by the EphemeralContainers feature flag.\ntype EphemeralContainer struct {\n\t// Ephemeral containers have all of the fields of Container, plus additional fields\n\t// specific to ephemeral containers. Fields in common with Container are in the\n\t// following inlined struct so than an EphemeralContainer may easily be converted\n\t// to a Container.\n\tEphemeralContainerCommon `json:\",inline\" protobuf:\"bytes,1,req\"`\n\n\t// If set, the name of the container from PodSpec that this ephemeral container targets.\n\t// The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container.\n\t// If not set then the ephemeral container is run in whatever namespaces are shared\n\t// for the pod. Note that the container runtime must support this feature.\n\t// +optional\n\tTargetContainerName string `json:\"targetContainerName,omitempty\" protobuf:\"bytes,2,opt,name=targetContainerName\"`\n}\n```\n\nMuch of the utility of Ephemeral Containers comes from the ability to run a\ncontainer within the PID namespace of another container. `TargetContainerName`\nallows targeting a container that doesn't share its PID namespace with the rest\nof the pod. We must modify the CRI to enable this functionality (see below).\n\n`EphemeralContainerCommon` is an inline copy of `Container` that resolves the\nfollowing contradictory requirements:\n\n1. Ephemeral containers should be represented by a type that is easily\n   convertible to `Container` so that code that operations on `Container` can\n   also operate on ephemeral containers.\n1. Fields of `Container` that have different behavior for ephemeral containers\n   should be separately and clearly documented. Since many fields of ephemeral\n   containers have different behavior, this requires a separate type.\n\n`EphemeralContainerCommon` contains fields that ephemeral containers have in\ncommon with `Container`. It's field-for-field copy of `Container`, which is\nenforced by the compiler:\n\n```\n// EphemeralContainerCommon converts to Container. All fields must be kept in sync between\n// these two types.\nvar _ = Container(EphemeralContainerCommon{})\n```\n\nSince `EphemeralContainerCommon` is inlined, the API machinery hides this\ncomplexity from the end user, who sees a type, `EphemeralContainer` which has\nall of the fields of `Container` plus an additional field `targetContainerName`.\n\n##### Alternative Considered: Omitting TargetContainerName\n\nIt would be simpler for the API, kubelet and kubectl if `EphemeralContainers`\nwas a `[]Container`, but as isolated PID namespaces will be the default for some\ntime, being able to target a container will provide a better user experience.\n\n#### Updating a Pod\n\nMost fields of `Pod.Spec` are immutable once created. There is a short whitelist\nof fields which may be updated, and we will extend this to include\n`EphemeralContainers`. The ability to add new containers is a large change for\nPod, however, and we'd like to begin conservatively by enforcing the following\nbest practices:\n\n1.  Ephemeral Containers lack guarantees for resources or execution, and they\n    will never be automatically restarted. To avoid pods that depend on\n    Ephemeral Containers, we allow their addition only in pod updates and\n    disallow them during pod create.\n1.  Some fields of `v1.Container` imply a fundamental role in a pod. We will\n    disallow the following fields in Ephemeral Containers: `ports`,\n    `livenessProbe`, `readinessProbe`, and `lifecycle.`\n1.  Some fields of `v1.Container` imply consequences for the entire pod. For\n    example, one would expect setting `resources` to increase resources\n    allocated to the pod, but this is not yet supported. We will disallow\n    `resources` in Ephemeral Containers.\n1.  Cluster administrators may want to restrict access to Ephemeral Containers\n    independent of other pod updates.\n\nTo enforce these restrictions and enable RBAC, we will introduce a new Pod\nsubresource, `/ephemeralcontainers`. `EphemeralContainers` can only be modified\nvia this subresource. `EphemeralContainerStatuses` is updated in the same manner\nas everything else in `Pod.Status` via `/status`.\n\n`Pod.Spec.EphemeralContainers` may be updated via `/ephemeralcontainers` as per\nnormal (using PUT, PATCH, etc) except that existing Ephemeral Containers may not\nbe modified or deleted. Deleting Ephemeral Containers is not supported in the\ninitial implementation to reduce complexity. It could be added in the future,\nbut see *Killing Ephemeral Containers* below for additional constraints.\n\nThe subresources `attach`, `exec`, `log`, and `portforward` are available for\nEphemeral Containers and will be forwarded by the apiserver. This means `kubectl\nattach`, `kubelet exec`, `kubectl log`, and `kubectl port-forward` will work for\nEphemeral Containers.\n\nOnce the pod is updated, the kubelet worker watching this pod will launch the\nEphemeral Container and update its status. A client creating a new Ephemeral\nContainer is expected to watch for the creation of the container status before\nattaching to the console using the existing attach endpoint,\n`/api/v1/namespaces/$NS/pods/$POD_NAME/attach`. Note that any output of the new\ncontainer occurring between its creation and attach will not be replayed, but it\ncan be viewed using `kubectl log`.\n\n### Container Runtime Interface (CRI) changes\n\nSince Ephemeral Containers use the Container Runtime Interface, Ephemeral\nContainers will work for any runtime implementing the CRI, including Windows\ncontainers. It's worth noting that Ephemeral Containers are significantly more\nuseful when the runtime implements [Process Namespace Sharing].\nWindows Server 2019 does not support process namespace sharing\n(see [doc](https://kubernetes.io/docs/setup/windows/intro-windows-in-kubernetes/#v1-pod)).\n\nThe CRI requires no changes for basic functionality, but it will need to be\nupdated to support container namespace targeting, described fully in\n[Targeting a Namespace].\n\n[Process Namespace Sharing]: https://git.k8s.io/enhancements/keps/sig-node/20190920-pod-pid-namespace.md\n[Targeting a Namespace]: https://git.k8s.io/enhancements/keps/sig-node/20190920-pod-pid-namespace.md#targeting-a-specific-containers-namespace\n\n### Creating Ephemeral Containers\n\n1.  A client constructs an `EphemeralContainer` based on command line and\n    and appends it to `Pod.Spec.EphemeralContainers`. It updates the pod using\n    the pod's `/ephemeralcontainers` subresource.\n1.  The apiserver validates and performs the pod update.\n    1.  Pod validation fails if container spec contains fields disallowed for\n        Ephemeral Containers or the same name as a container in the spec or\n        `EphemeralContainers`.\n    1.  API resource versioning resolves update races.\n1.  The kubelet's pod watcher notices the update and triggers a `syncPod()`.\n    During the sync, the kubelet calls `kuberuntime.StartEphemeralContainer()`\n    for any new Ephemeral Container.\n    1.  `StartEphemeralContainer()` uses the existing `startContainer()` to\n        start the Ephemeral Container.\n    1.  After initial creation, future invocations of `syncPod()` will publish\n        its ContainerStatus but otherwise ignore the Ephemeral Container. It\n        will exist for the life of the pod sandbox or it exits. In no event will\n        it be restarted.\n1.  `syncPod()` finishes a regular sync, publishing an updated PodStatus (which\n    includes the new `EphemeralContainer`) by its normal, existing means.\n1.  The client performs an attach to the debug container's console.\n\nThere are no limits on the number of Ephemeral Containers that can be created in\na pod, but exceeding a pod's resource allocation may cause the pod to be\nevicted.\n\n### Restarting and Reattaching Ephemeral Containers\n\nEphemeral Containers will not be restarted.\n\nWe want to be more user friendly by allowing re-use of the name of an exited\nephemeral container, but this will be left for a future improvement.\n\nOne can reattach to a Ephemeral Container using `kubectl attach`. When supported\nby a runtime, multiple clients can attach to a single debug container and share\nthe terminal. This is supported by Docker.\n\n### Killing Ephemeral Containers\n\nEphemeral Containers will not be killed automatically unless the pod is\ndestroyed.  Ephemeral Containers will stop when their command exits, such as\nexiting a shell.  Unlike `kubectl exec`, processes in Ephemeral Containers will\nnot receive an EOF if their connection is interrupted.\n\nA future improvement could allow killing Ephemeral Containers when they're\nremoved from `EphemeralContainers`, but it's not clear that we want to allow\nthis. Removing an Ephemeral Container spec makes it unavailable for future\nauthorization decisions (e.g. whether to authorize exec in a pod that had a\nprivileged Ephemeral Container).\n\n### User Stories\n\n#### Operations\n\nJonas runs a service \"neato\" that consists of a statically compiled Go binary\nrunning in a minimal container image. One of the its pods is suddenly having\ntrouble connecting to an internal service. Being in operations, Jonas wants to\nbe able to inspect the running pod without restarting it, but he doesn't\nnecessarily need to enter the container itself. He wants to:\n\n1.  Inspect the filesystem of target container\n1.  Execute debugging utilities not included in the container image\n1.  Initiate network requests from the pod network namespace\n\nThis is achieved by running a new \"debug\" container in the pod namespaces. His\ntroubleshooting session might resemble:\n\n```\n% kubectl debug -it -m debian neato-5thn0 -- bash\nroot@debug-image:~# ps x\n  PID TTY      STAT   TIME COMMAND\n    1 ?        Ss     0:00 /pause\n   13 ?        Ss     0:00 bash\n   26 ?        Ss+    0:00 /neato\n  107 ?        R+     0:00 ps x\nroot@debug-image:~# cat /proc/26/root/etc/resolv.conf\nsearch default.svc.cluster.local svc.cluster.local cluster.local\nnameserver 10.155.240.10\noptions ndots:5\nroot@debug-image:~# dig @10.155.240.10 neato.svc.cluster.local.\n\n; \u003c\u003c\u003e\u003e DiG 9.9.5-9+deb8u6-Debian \u003c\u003c\u003e\u003e @10.155.240.10 neato.svc.cluster.local.\n; (1 server found)\n;; global options: +cmd\n;; connection timed out; no servers could be reached\n```\n\nJonas discovers that the cluster's DNS service isn't responding.\n\n#### Debugging\n\nThurston is debugging a tricky issue that's difficult to reproduce. He can't\nreproduce the issue with the debug build, so he attaches a debug container to\none of the pods exhibiting the problem:\n\n```\n% kubectl debug -it --image=gcr.io/neato/debugger neato-5x9k3 -- sh\nDefaulting container name to debug.\n/ # ps x\nPID   USER     TIME   COMMAND\n    1 root       0:00 /pause\n   13 root       0:00 /neato\n   26 root       0:00 sh\n   32 root       0:00 ps x\n/ # gdb -p 13\n...\n```\n\nHe discovers that he needs access to the actual container, which he can achieve\nby installing busybox into the target container:\n\n```\nroot@debug-image:~# cp /bin/busybox /proc/13/root\nroot@debug-image:~# nsenter -t 13 -m -u -p -n -r /busybox sh\n\n\nBusyBox v1.22.1 (Debian 1:1.22.0-9+deb8u1) built-in shell (ash)\nEnter 'help' for a list of built-in commands.\n\n/ # ls -l /neato\n-rwxr-xr-x    2 0        0           746888 May  4  2016 /neato\n```\n\nNote that running the commands referenced above requires `CAP_SYS_ADMIN` and\n`CAP_SYS_PTRACE`.\n\nThis scenario also requires process namespace sharing which is not available\non Windows.\n\n#### Automation\n\nGinger is a security engineer tasked with running security audits across all of\nher company's running containers. Even though her company has no standard base\nimage, she's able to audit all containers using:\n\n```\n% for pod in $(kubectl get -o name pod); do\n    kubectl debug -m gcr.io/neato/security-audit -p $pod /security-audit.sh\n  done\n```\n\n#### Technical Support\n\nRoy's team provides support for his company's multi-tenant cluster. He can\naccess the Kubernetes API (as a viewer) on behalf of the users he's supporting,\nbut he does not have administrative access to nodes or a say in how the\napplication image is constructed. When someone asks for help, Roy's first step\nis to run his team's autodiagnose script:\n\n```\n% kubectl debug --image=k8s.gcr.io/autodiagnose nginx-pod-1234\n```\n\n### Implementation Details/Notes/Constraints\n\n1.  There are no guaranteed resources for ad-hoc troubleshooting. If\n    troubleshooting causes a pod to exceed its resource limit it may be evicted.\n1.  There's an output stream race inherent to creating then attaching a\n    container which causes output generated between the start and attach to go\n    to the log rather than the client. This is not specific to Ephemeral\n    Containers and exists because Kubernetes has no mechanism to attach a\n    container prior to starting it. This larger issue will not be addressed by\n    Ephemeral Containers, but Ephemeral Containers would benefit from future\n    improvements or work arounds.\n1.  Ephemeral Containers should not be used to build services, which we've\n    attempted to reflect in the API.\n\n### Risks and Mitigations\n\n#### Security Considerations\n\nEphemeral Containers have no additional privileges above what is available to\nany `v1.Container`. It's the equivalent of configuring an shell container in a\npod spec except that it is created on demand.\n\nAdmission plugins must be updated to guard `/ephemeralcontainers`. They should\napply the same container image and security policy as for regular containers.\n\nWe designed the API to be compatible with the existing Kubernetes RBAC\nmechanism. Cluster Administrators are able to authorize Ephemeral Containers\nindependent of other pod operations.\n\nWe've worked with the sig-auth leads to review these changes.\n\n#### Requiring a Subresource\n\nIt would simplify initial implementation if we updated `EphemeralContainers`\nwith a standard pod update, but we've received clear feedback that cluster\nadministrators want close control over this feature. This requires a separate\nsubresource.\n\nThis feature will have a long alpha, and we can re-examine this decision prior\nto exiting alpha.\n\n#### Creative New Uses of Ephemeral Containers\n\nThough this KEP focuses on debugging, Ephemeral Containers are a general\naddition to Kubernetes, and we should expect that the community will use them to\nsolve other problems. This is good and intentional, but Ephemeral Containers\nhave inherent limitations which can lead to pitfalls.\n\nFor example, it might be tempting to use Ephemeral Containers to perform\ncritical but asynchronous functions like backing up a production database, but\nthis would be dangerous because Ephemeral Containers have no execution\nguarantees and could even cause the database pod to be evicted by exceeding its\nresource allocation.\n\nAs much as possible we've attempted to make it clear in the API these\nlimitations, and we've restricted the use of fields that imply a container\nshould be part of `Spec.Containers`.\n\n## Design Details\n\n### Test Plan\n\nThis feature will be tested with a combination of unit, integration and e2e\ntests. In particular:\n\n* Field validation (e.g. of Container fields disallowed in Ephemeral Containers)\n  will be tested in unit tests.\n* Pod update semantics will be tested in integration tests.\n* Ephemeral Container creation will be tested in e2e-node.\n\nNone of the tests for this feature are unusual or tricky.\n\n### Graduation Criteria\n\n#### Alpha -\u003e Beta Graduation\n\n- [ ] Ephemeral Containers API has been in alpha for at least 2 releases.\n- [ ] Ephemeral Containers support namespace targeting.\n- [ ] Tests are in Testgrid and linked in KEP.\n- [ ] Metrics for Ephemeral Containers are added to existing contain creation\n  metrics.\n- [ ] CLI using Ephemeral Containers for debugging checked into a Kubernetes\n  project repository (e.g. in `kubectl` or a `kubectl` plugin).\n- [ ] A task on https://kubernetes.io/docs/tasks/ describes how to troubleshoot\n  a running pod using Ephemeral Containers.\n- [ ] A survey sent to early adopters doesn't reveal any major shortcomings.\n\n#### Beta -\u003e GA Graduation\n\n- [ ] Ephemeral Containers have been in beta for at least 2 releases.\n- [ ] Ephemeral Containers see use in 3 projects or articles.\n\n### Version Skew Strategy\n\nFor API compatibility, we rely on the [Adding Unstable Features to Stable\nVersions] API Changes recommendations. An n-2 kubelet won't recognize the new\nfields, so the API should remain in alpha for at least 2 releases.\n\nNamespace targeting requires adding an enum value to the CRI. This will present\nan unknown value to old CRIs. Ideally, [CRI Optional Runtime Features] would\nallow us to query for this feature, but this is unlikely to be implemented.\nInstead, we will update the CRI and add a conformance test. (As of this KEP the\nCRI is still in alpha.) Runtimes will be expected to handle an unknown\n`NamespaceMode` gracefully.\n\n[Adding Unstable Features to Stable Versions]: https://git.k8s.io/community/contributors/devel/sig-architecture/api_changes.md#adding-unstable-features-to-stable-versions\n[CRI Optional Runtime Features]: https://issues.k8s.io/32803\n\n## Implementation History\n\n- *2016-06-09*: Opened [#27140](https://issues.k8s.io/27140) to explore\n  solutions for debugging minimal container images.\n- *2017-09-27*: Merged first version of proposal for troubleshooting running\n  pods [kubernetes/community#649](https://github.com/kubernetes/community/pull/649)\n- *2018-08-23*: Merged update to use `Container` in `Pod.Spec`\n  [kubernetes/community#1269](https://github.com/kubernetes/community/pull/1269)\n- *2019-02-12*: Ported design proposal to KEP.\n- *2019-04-24*: Added notes on Windows feature compatibility\n\n## Alternatives\n\nWe've explored many alternatives to Ephemeral Containers for the purposes of\ndebugging, so this section is quite long.\n\n### Container Spec in PodStatus\n\nOriginally there was a desire to keep the pod spec immutable, so we explored\nmodifying only the pod status. An `EphemeralContainer` would contain a Spec, a\nStatus and a Target:\n\n```\n// EphemeralContainer describes a container to attach to a running pod for troubleshooting.\ntype EphemeralContainer struct {\n        metav1.TypeMeta `json:\",inline\"`\n\n        // Spec describes the Ephemeral Container to be created.\n        Spec *Container `json:\"spec,omitempty\" protobuf:\"bytes,2,opt,name=spec\"`\n\n        // Most recently observed status of the container.\n        // This data may not be up to date.\n        // Populated by the system.\n        // Read-only.\n        // +optional\n        Status *ContainerStatus `json:\"status,omitempty\" protobuf:\"bytes,3,opt,name=status\"`\n\n        // If set, the name of the container from PodSpec that this ephemeral container targets.\n        // If not set then the ephemeral container is run in whatever namespaces are shared\n        // for the pod.\n        TargetContainerName string `json:\"targetContainerName,omitempty\" protobuf:\"bytes,4,opt,name=targetContainerName\"`\n}\n```\n\nEphemeral Containers for a pod would be listed in the pod's status:\n\n```\ntype PodStatus struct {\n        ...\n        // List of user-initiated ephemeral containers that have been run in this pod.\n        // +optional\n        EphemeralContainers []EphemeralContainer `json:\"ephemeralContainers,omitempty\" protobuf:\"bytes,11,rep,name=ephemeralContainers\"`\n\n}\n```\n\nTo create a new Ephemeral Container, one would append a new `EphemeralContainer`\nwith the desired `v1.Container` as `Spec` in `Pod.Status` and updates the `Pod`\nin the API. Users cannot normally modify the pod status, so we'd create a new\nsubresource `/ephemeralcontainers` that allows an update of solely\n`EphemeralContainers` and enforces append-only semantics.\n\nSince we have a requirement to describe the Ephemeral Container with a\n`v1.Container`, this lead to a \"spec in status\" that seemed to violate API best\npractices. It was confusing, and it required added complexity in the kubelet to\npersist and publish user intent, which is rightfully the job of the apiserver.\n\n### Extend the Existing Exec API (\"exec++\")\n\nA simpler change is to extend `v1.Pod`'s `/exec` subresource to support\n\"executing\" container images. The current `/exec` endpoint must implement `GET`\nto support streaming for all clients. We don't want to encode a (potentially\nlarge) `v1.Container` into a query string, so we must extend `v1.PodExecOptions`\nwith the specific fields required for creating a Debug Container:\n\n```\n// PodExecOptions is the query options to a Pod's remote exec call\ntype PodExecOptions struct {\n        ...\n        // EphemeralContainerName is the name of an ephemeral container in which the\n        // command ought to be run. Either both EphemeralContainerName and\n        // EphemeralContainerImage fields must be set, or neither.\n        EphemeralContainerName *string `json:\"ephemeralContainerName,omitempty\" ...`\n\n        // EphemeralContainerImage is the image of an ephemeral container in which the command\n        // ought to be run. Either both EphemeralContainerName and EphemeralContainerImage\n        // fields must be set, or neither.\n        EphemeralContainerImage *string `json:\"ephemeralContainerImage,omitempty\" ...`\n}\n```\n\nAfter creating the Ephemeral Container, the kubelet would upgrade the connection\nto streaming and perform an attach to the container's console. If disconnected,\nthe Ephemeral Container could be reattached using the pod's `/attach` endpoint\nwith `EphemeralContainerName`.\n\nEphemeral Containers could not be removed via the API and instead the process\nmust terminate. While not ideal, this parallels existing behavior of `kubectl\nexec`. To kill an Ephemeral Container one would `attach` and exit the process\ninteractively or create a new Ephemeral Container to send a signal with\n`kill(1)` to the original process.\n\nSince the user cannot specify the `v1.Container`, this approach sacrifices a\ngreat deal of flexibility. This solution still requires the kubelet to publish a\n`Container` spec in the `PodStatus` that can be examined for future admission\ndecisions and so retains many of the downsides of the Container Spec in\nPodStatus approach.\n\n### Ephemeral Container Controller\n\nKubernetes prefers declarative APIs where the client declares a state for\nKubernetes to enact. We could implement this in a declarative manner by creating\na new `EphemeralContainer` type:\n\n```\ntype EphemeralContainer struct {\n        metav1.TypeMeta\n        metav1.ObjectMeta\n\n        Spec v1.Container\n        Status v1.ContainerStatus\n}\n```\n\nA new controller in the kubelet would watch for EphemeralContainers and\ncreate/delete debug containers. `EphemeralContainer.Status` would be updated by\nthe kubelet at the same time it updates `ContainerStatus` for regular and init\ncontainers. Clients would create a new `EphemeralContainer` object, wait for it\nto be started and then attach using the pod's attach subresource and the name of\nthe `EphemeralContainer`.\n\nA new controller is a significant amount of complexity to add to the kubelet,\nespecially considering that the kubelet is already watching for changes to pods.\nThe kubelet would have to be modified to create containers in a pod from\nmultiple config sources. SIG Node strongly prefers to minimize kubelet\ncomplexity.\n\n### Mutable Pod Spec Containers\n\nRather than adding to the pod API, we could instead make the pod spec mutable so\nthe client can generate an update adding a container. `SyncPod()` has no issues\nadding the container to the pod at that point, but an immutable pod spec has\nbeen a basic assumption and best practice in Kubernetes. Changing this\nassumption complicates the requirements of the kubelet state machine. Since the\nkubelet was not written with this in mind, we should expect such a change would\ncreate bugs we cannot predict.\n\n### Image Exec\n\nAn earlier version of this proposal suggested simply adding `Image` parameter to\nthe exec API. This would run an ephemeral container in the pod namespaces\nwithout adding it to the pod spec or status. This container would exist only as\nlong as the process it ran. This parallels the current kubectl exec, including\nits lack of transparency. We could add constructs to track and report on both\ntraditional exec process and exec containers. In the end this failed to meet our\ntransparency requirements.\n\n### Attaching Container Type Volume\n\nCombining container volumes ([#831](https://issues.k8s.io/831)) with the ability\nto add volumes to the pod spec would get us most of the way there. One could\nmount a volume of debug utilities at debug time. Docker does not allow adding a\nvolume to a running container, however, so this would require a container\nrestart. A restart doesn't meet our requirements for troubleshooting.\n\nRather than attaching the container at debug time, kubernetes could always\nattach a volume at a random path at run time, just in case it's needed. Though\nthis simplifies the solution by working within the existing constraints of\n`kubectl exec`, it has a sufficient list of minor limitations (detailed in\n[#10834](https://issues.k8s.io/10834)) to result in a poor user experience.\n\n### Using docker cp and exec\n\nInstead of creating an additional container with a different image, `docker cp`\ncould be used to add binaries into a running container before calling `exec` on\nthe process. This approach would be feasible on Windows as it doesn't require\nprocess namespace sharing. It also doesn't involve the complexities with adding\nmounts as described in [Attaching Container Type Volume](#attaching-container-type-volume).\nHowever, it doesn't provide a convenient way to package or distribute binaries\nas described in this KEP or the alternate [Image Exec](#image-exec) proposal.\n`docker cp` also doesn't have a CRI equivalent, so that would need to be\naddressed in an alternate proposal.\n\n### Inactive container\n\nIf Kubernetes supported the concept of an \"inactive\" container, we could\nconfigure it as part of a pod and activate it at debug time. In order to avoid\ncoupling the debug tool versions with those of the running containers, we would\nwant to ensure the debug image was pulled at debug time. The container could\nthen be run with a TTY and attached using kubectl.\n\nThe downside of this approach is that it requires prior configuration. In\naddition to requiring prior consideration, it would increase boilerplate config.\nA requirement for prior configuration makes it feel like a workaround rather\nthan a feature of the platform.\n\n### Implicit Empty Volume\n\nKubernetes could implicitly create an EmptyDir volume for every pod which would\nthen be available as a target for either the kubelet or a sidecar to extract a\npackage of binaries.\n\nUsers would have to be responsible for hosting a package build and distribution\ninfrastructure or rely on a public one. The complexity of this solution makes it\nundesirable.\n\n### Standalone Pod in Shared Namespace (\"Debug Pod\")\n\nRather than inserting a new container into a pod namespace, Kubernetes could\ninstead support creating a new pod with container namespaces shared with\nanother, target pod. This would be a simpler change to the Kubernetes API, which\nwould only need a new field in the pod spec to specify the target pod. To be\nuseful, the containers in this \"Debug Pod\" should be run inside the namespaces\n(network, pid, etc) of the target pod but remain in a separate resource group\n(e.g. cgroup for container-based runtimes).\n\nThis would be a rather large change for pod, which is currently treated as an\natomic unit. The Container Runtime Interface has no provisions for sharing\noutside of a pod sandbox and would need a refactor. This could be a complicated\nchange for non-container runtimes (e.g. hypervisor runtimes) which have more\nrigid boundaries between pods.\n\nThis is pushing the complexity of the solution from the kubelet to the runtimes.\nMinimizing change to the Kubernetes API is not worth the increased complexity\nfor the kubelet and runtimes.\n\nIt could also be possible to implement a Debug Pod as a privileged pod that runs\nin the host namespace and interacts with the runtime directly to run a new\ncontainer in the appropriate namespace. This solution would be runtime-specific\nand pushes the complexity of debugging to the user. Additionally, requiring\nnode-level access to debug a pod does not meet our requirements.\n\n### Exec from Node\n\nThe kubelet could support executing a troubleshooting binary from the node in\nthe namespaces of the container. Once executed this binary would lose access to\nother binaries from the node, making it of limited utility and a confusing user\nexperience.\n\nThis couples the debug tools with the lifecycle of the node, which is worse than\ncoupling it with container images.\n"
+  },
+  {
+    "id": "9c0447f5980da502ccc5fd7a4b658224",
+    "title": "Add pod-startup liveness-probe holdoff for slow-starting pods",
+    "authors": ["@matthyx"],
+    "owningSig": "sig-node",
+    "participatingSigs": ["sig-apps", "sig-architecture"],
+    "reviewers": ["@thockin"],
+    "approvers": ["@derekwaynecarr", "@thockin"],
+    "editor": "TBD",
+    "creationDate": "2019-02-21",
+    "lastUpdated": "2019-05-18",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Add pod-startup liveness-probe holdoff for slow-starting pods\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Details](#implementation-details)\n  - [Why a new probe instead of initializationFailureThreshold](#why-a-new-probe-instead-of-initializationfailurethreshold)\n  - [Configuration example](#configuration-example)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Feature Gate](#feature-gate)\n  - [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [X] KEP approvers have set the KEP status to `implementable`\n- [X] Design details are appropriately documented\n- [X] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [X] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [X] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nSlow starting containers are difficult to address with the current status of health probes: they are either killed before being up, or could be left deadlocked during a very long time before being killed.\n\nThis proposal adds a new probe called `startupProbe` that holds off all the other probes until the pod has finished its startup. In the case of a slow-starting pod, it could poll on a relatively short period with a high `failureThreshold`. Once it is satisfied, the other probes can start.\n\n## Motivation\n\nSlow starting containers here refer to containers that require a significant amount of time (one to several minutes) to start. There can be various reasons for this slow startup:\n\n- long data initialization: only the first startup takes a lot of time\n- heavy workload: every startups take a lot of time\n- underpowered/overloaded node: startup times depend on external factors (however, solving node related issues is not a goal of this proposal)\n\nThe main problem with this kind containers is that they should be given enough time to start before having `livenessProbe` fail `failureThreshold` times, which triggers a kill by the `kubelet` before they have a chance to be up.\n\nThere are various strategies to handle this situation with the current API:\n\n- Delay the initial `livenessProbe` sufficiently to permit the container to start up (set `initialDelaySeconds` greater than **startup time**). While this ensures no `livenessProbe` will run and fail during the startup period (triggering a kill), it also delays deadlock detection if the container starts faster than `initialDelaySeconds`. Also, since the `livenessProbe` isn't run at all during startup, there is no feedback loop on the actual startup time of the container.\n- Increase the allowed number of `livenessProbe` failures until `kubelet` kills the container (set `failureThreshold` so that `failureThreshold` times `periodSeconds` is greater than **startup time**). While this gives enough time for the container to start up and allows a feedback loop, it prevents the container from being killed in a timely manner if it deadlocks or otherwise hangs after it has initially successfully come up.\n\nHowever, none of these strategies provide an timely answer to slow starting containers stuck in a deadlock, which is the primary reason of setting up a `livenessProbe`.\n\n### Goals\n\n- Allow slow starting containers to run safely during startup with health probes enabled.\n- Improve documentation of the `Probe` structure in core types' API.\n- Improve `kubernetes.io/docs` section about Pod lifecycle:\n  - Clearly state that PostStart handlers do not delay probe executions.\n  - Introduce and explain this new probe.\n  - Document appropriate use cases for this new probe.\n\n### Non-Goals\n\n- This proposal does not address the issue of pod load affecting startup (or any other probe that may be delayed due to load). It is acting strictly at the pod level, not the node level.\n- This proposal will only update the official Kubernetes documentation, excluding [A Pod's Life] and other well referenced pages explaining probes.\n\n[A Pod's Life]: https://blog.openshift.com/kubernetes-pods-life/\n\n## Proposal\n\n### Implementation Details\n\nThe proposed solution is to add a new probe named `startupProbe` in the container spec of a pod which will determine whether it has finished starting up.\n\nIt also requires keeping the state of the container (has the `startupProbe` ever succeeded?) using a boolean `Started` inside the ContainerStatus struct.\n\nDepending on `Started` the probing mechanism in `worker.go` might be altered:\n\n- `Started == true`: the kubelet worker works the same way as today\n- `Started == false`: the kubelet worker only probes the `startupProbe`\n\nIf `startupProbe` fails more than `failureThreshold` times, the result is the same as today when `livenessProbe` fails: the container is killed and might be restarted depending on `restartPolicy`.\n\nIf no `startupProbe` is defined, `Started` is initialized with `true`.\n\n### Why a new probe instead of initializationFailureThreshold\n\nWhile trying to merge PR [#1014] in time for code-freeze, @thockin has make the following points which I agree with:\n\n\u003e I feel pretty strongly that something like a startupProbe would be net simpler to comprehend than a new field on liveness.\n\u003e\n\u003e In [issuecomment-437208330] we looked at a different take on this API - it is more precise in its meaning and rather than add yet another behavior modifier to probe, it can reuse the probe structure directly.\n\nHere is the excerpt of [issuecomment-437208330] talking about the design:\n\n\u003e An idea that I toyed with but never pursued was a StartupProbe - all the other probes would wait on it at pod startup. It could poll on a relatively short period with a long FailureThreshold. Once it is satisfied, the other probes can start.\n\nI also think the third probe gives more flexibility if we find other good reasons to inhibit `livenessProbe` or `readinessProbe` before something occurs during container startup.\n\n[#1014]: https://github.com/kubernetes/enhancements/pull/1014\n[issuecomment-437208330]: https://github.com/kubernetes/kubernetes/issues/27114#issuecomment-437208330\n\n### Configuration example\n\nThis example shows how startupProbe can be used to emulate the functionality of `initializationFailureThreshold` as it was proposed before:\n\n```yaml\nports:\n- name: liveness-port\n  containerPort: 8080\n  hostPort: 8080\n\nlivenessProbe:\n  httpGet:\n    path: /healthz\n    port: liveness-port\n  failureThreshold: 1\n  periodSeconds: 10\n\nstartupProbe:\n  httpGet:\n    path: /healthz\n    port: liveness-port\n  failureThreshold: 30 (=initializationFailureThreshold)\n  periodSeconds: 10\n```\n\n## Design Details\n\n### Test Plan\n\nUnit tests will be implemented with `newTestWorker` and will check the following:\n\n- proper initialization of `Started` to false\n- `Started` becomes true as soon as `startupProbe` succeeds\n- `livenessProbe` and `readinessProbe` are disabled until `Started` is true\n- `startupProbe` is disabled after `Started` becomes true\n- `failureThreshold` exceeded for `startupProbe` kills the container\n\nE2e tests will also cover the main use-case for this probe:\n\n- `startupProbe` disables `livenessProbe` long enough to simulate a slow starting container, using a high `failureThreshold`\n\n### Feature Gate\n\n- Expected feature gate key: `StartupProbeEnabled`\n- Expected default value: `false`\n\n### Graduation Criteria\n\n- Alpha: Initial support for `startupProbe` added. Disabled by default.\n- Beta: `startupProbe` enabled with no default configuration.\n- Stable: `startupProbe` enabled with no default configuration.\n\n## Implementation History\n\n- 2018-11-27: prototype implemented in PR [#71449] under review\n- 2019-03-05: present KEP to sig-node\n- 2019-04-11: open issue in enhancements [#950]\n- 2019-05-01: redesign to additional probe after @thockin [proposal]\n- 2019-05-02: add test plan\n- 2019-05-13: redesign implemented in new PR [#77807]\n- 2019-05-13: related documentation added in PR [#14297]\n\n[#71449]: https://github.com/kubernetes/kubernetes/pull/71449\n[#950]: https://github.com/kubernetes/enhancements/issues/950\n[proposal]: https://github.com/kubernetes/kubernetes/issues/27114#issuecomment-437208330\n[#77807]: https://github.com/kubernetes/kubernetes/pull/77807\n[#14297]: https://github.com/kubernetes/website/pull/14297\n"
+  },
+  {
+    "id": "4d91cbec4a34190b790059ad06155364",
+    "title": "Pod Overhead",
+    "authors": ["@egernst"],
+    "owningSig": "sig-node",
+    "participatingSigs": ["sig-scheduling", "sig-autoscaling", "sig-windows"],
+    "reviewers": ["@tallclair", "@derekwaynecarr", "@dchen1107"],
+    "approvers": ["TBD"],
+    "editor": "TBD",
+    "creationDate": "2019-02-26",
+    "lastUpdated": "2019-04-12",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Pod Overhead\n\nThis includes the Summary and Motivation sections.\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [API Design](#api-design)\n    - [Pod overhead](#pod-overhead-1)\n    - [Container Runtime Interface (CRI)](#container-runtime-interface-cri)\n  - [ResourceQuota changes](#resourcequota-changes)\n  - [RuntimeClass changes](#runtimeclass-changes)\n  - [RuntimeClass admission controller](#runtimeclass-admission-controller)\n  - [Implementation Details](#implementation-details)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Drawbacks](#drawbacks)\n- [Alternatives](#alternatives)\n  - [Introduce pod level resource requirements](#introduce-pod-level-resource-requirements)\n  - [Leaving the PodSpec unchanged](#leaving-the-podspec-unchanged)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those\napprovers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nSandbox runtimes introduce a non-negligible overhead at the pod level which must be accounted for\neffective scheduling, resource quota management, and constraining.\n\n## Motivation\n\nPods have some resource overhead. In our traditional linux container (Docker) approach,\nthe accounted overhead is limited to the infra (pause) container, but also invokes some\noverhead accounted to various system components including: Kubelet (control loops), Docker,\nkernel (various resources), fluentd (logs). The current approach is to reserve a chunk\nof resources for the system components (system-reserved, kube-reserved, fluentd resource\nrequest padding), and ignore the (relatively small) overhead from the pause container, but\nthis approach is heuristic at best and doesn't scale well.\n\nWith sandbox pods, the pod overhead potentially becomes much larger, maybe O(100mb). For\nexample, Kata containers must run a guest kernel, kata agent, init system, etc. Since this\noverhead is too big to ignore, we need a way to account for it, starting from quota enforcement\nand scheduling.\n\n### Goals\n\n* Provide a mechanism for accounting pod overheads which are specific to a given runtime solution\n\n### Non-Goals\n\n* making runtimeClass selections\n* auto-detecting overhead\n* per-container overhead\n* creation of pod-level resource requirements\n\n## Proposal\n\nAugment the RuntimeClass definition and the `PodSpec` to introduce\nthe field `Overhead *ResourceList`. This field represents the overhead associated\nwith running a pod for a given runtimeClass.  A mutating admission controller is\nintroduced which will update the `Overhead` field in the workload's `PodSpec` to match\nwhat is provided for the selected RuntimeClass, if one is specified.\n\nKubelet's creation of the pod cgroup will be calculated as the sum of container\n`ResourceRequirements.Limits` fields, plus the Overhead associated with the pod.\n\nThe scheduler, resource quota handling, and Kubelet's pod cgroup creation and eviction handling\nwill take Overhead into account, as well as the sum of the pod's container requests.\n\nHorizontal and Veritical autoscaling are calculated based on container level statistics,\nso should not be impacted by pod Overhead.\n\n### API Design\n\n#### Pod overhead\nIntroduce a Pod.Spec.Overhead field on the pod to specify the pods overhead.\n\n```\nPod {\n  Spec PodSpec {\n    // Overhead is the resource overhead incurred from the runtime.\n    // +optional\n    Overhead *ResourceList\n  }\n}\n```\n\nAll PodSpec and RuntimeClass fields are immutable, including the `Overhead` field. For scheduling,\nthe pod `Overhead` is added to the container resource requests.\n\nWe don't currently enforce resource limits on the pod cgroup, but this becomes feasible once pod\noverhead is accountable. If the pod specifies an overhead, and all containers in the pod specify a\nlimit, then the sum of those limits and overhead becomes a pod-level limit, enforced through the pod\ncgroup.\n\nUsers are not expected to manually set `Overhead`; any prior values being set will result in the workload\nbeing rejected. If runtimeClass is configured and selected in the PodSpec, `Overhead` will be set to the value\ndefined in the corresponding runtimeClass. This is described in detail in\n[RuntimeClass admission controller](#runtimeclass-admission-controller).\n\nBeing able to specify resource requirements for a workload at the pod level instead of container\nlevel has been discussed, but isn't proposed in this KEP.\n\nIn the event that pod-level requirements are introduced, pod overhead should be kept separate. This\nsimplifies several scenarios:\n - overhead, once added to the spec, stays with the workload, even if runtimeClass is redefined\n or removed.\n - the pod spec can be referenced directly from scheduler, resourceQuota controller and kubelet,\n instead of referencing a runtimeClass object which could have possibly been removed.\n\n#### Container Runtime Interface (CRI)\n\nThe pod cgroup is managed by the Kubelet, so passing the pod-level resource to the CRI implementation\nis not strictly necessary. However, some runtimes may wish to take advantage of this information, for\ninstance for sizing the Kata Container VM.\n\nLinuxContainerResources is added to the LinuxPodSandboxConfig for both overhead and container\ntotals, as optional fields:\n\n```\ntype LinuxPodSandboxConfig struct {\n\tOverhead *LinuxContainerResources\n\tContainerResources *LinuxContainerResources\n}\n```\n\nWindowsContainerResources is added to a newly created WindowsPodSandboxConfig for both overhead and container\ntotals, as optional fields:\n\n```\ntype WindowsPodSandboxConfig struct {\n\tOverhead *WindowsContainerResources\n\tContainerResources *WindowsContainerResources\n}\n```\n\nContainerResources field in the LinuxPodSandboxConfig and WindowsPodSandboxConfig matches the pod-level limits\n(i.e. total of container limits). Overhead is tracked separately since the sandbox overhead won't necessarily\nguide sandbox sizing, but instead used for better management of the resulting sandbox on the host.\n\n### ResourceQuota changes\n\nPod overhead will be counted against an entity's ResourceQuota. The controller will be updated to\nadd the pod `Overhead` to the container resource request summation.\n\n### RuntimeClass changes\n\nExpand the runtimeClass type to include sandbox overhead, `Overhead *Overhead`.\n\nWhere Overhead is defined as follows:\n\n```\ntype Overhead struct {\n  PodFixed *ResourceList\n}\n```\n\nIn the future, the `Overhead` definition could be extended to include fields that describe a percentage\nbased overhead (scale the overhead based on the size of the pod), or container-level overheads. These are\nleft out of the scope of this proposal.\n\n### RuntimeClass admission controller\n\nThe pod resource overhead must be defined prior to scheduling, and we shouldn't make the user\ndo it. To that end, we propose a mutating admission controller: RuntimeClass. This admission controller\nis also proposed for the [native RuntimeClass scheduling KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/runtime-class-scheduling.md).\n\nIn the scope of this KEP, The RuntimeClass controller will have a single job: set the pod overhead field in the\nworkload's PodSpec according to the runtimeClass specified.\n\nIt is expected that only the RuntimeClass controller will set Pod.Spec.Overhead. If a value is provided, the pod will\nbe rejected.\n\nGoing forward, I foresee additional controller scope around runtimeClass:\n - validating the runtimeClass selection: This would require applying some kind of pod-characteristic labels\n (runtimeClass selectors?) which would then be consumed by an admission controller and checked against known\n capabilities on a per runtimeClass basis. This is is beyond the scope of this proposal.\n - Automatic runtimeClass selection: A controller could exist which would attempt to automatically select the\n most appropriate runtimeClass for the given pod. This, again, is beyond the scope of this proposal.\n\n### Implementation Details\n\nWith this proposal, the following changes are required:\n - Add the new API to the pod spec and RuntimeClass\n - Update the RuntimeClass controller to merge the overhead into the pod spec\n - Update the ResourceQuota controller to account for overhead\n - Update the scheduler to account for overhead\n - Update the kubelet (admission, eviction, cgroup limits) to handle overhead\n\n### Risks and Mitigations\n\nThis proposal introduces changes across several Kubernetes components and a change in behavior *if* Overhead fields\nare utilized. To help mitigate this risk, I propose that this be treated as a new feature with an independent feature gate.\n\n## Design Details\n\n### Graduation Criteria\n\nThis KEP will be treated as a new feature, and will be introduced with a new feature gate, PodOverhead.\n\nPlan to introduce this utilizing maturity levels: alpha, beta and stable.\n\nGraduation criteria between these levels to be determined.\n\n### Upgrade / Downgrade Strategy\n\nIf applicable, how will the component be upgraded and downgraded? Make sure this is in the test plan.\n\nConsider the following in developing an upgrade/downgrade strategy for this enhancement:\n- What changes (in invocations, configurations, API use, etc.) is an existing cluster required to make on upgrade in order to keep previous behavior?\n- What changes (in invocations, configurations, API use, etc.) is an existing cluster required to make on upgrade in order to make use of the enhancement?\n\n### Version Skew Strategy\n\nSet the overhead to the max of the two version until the rollout is complete.  This may be more problematic\nif a new version increases (rather than decreases) the required resources.\n\n## Implementation History\n\n- 2019-04-04: Initial KEP published.\n\n## Drawbacks\n\nThis KEP introduces further complexity, and adds a field the PodSpec which users aren't expected to modify.\n\n## Alternatives\n\nIn order to achieve proper handling of sandbox runtimes, the scheduler/resourceQuota handling needs to take\ninto account the overheads associated with running a particular runtimeClass.\n\n### Introduce pod level resource requirements\n\nRather than just introduce overhead, add support for general pod-level resource requirements. Pod level\nresource requirements are useful for shared resources (hugepages, memory when doing emptyDir volumes).\n\nEven if this were to be introduced, there is a benefit in keeping the overhead separate.\n - post-pod creation handling of pod events: if runtimeClass definition is removed after a pod is created,\n  it will be very complicated to calculate which part of the pod resource requirements were associated with\n  the workloads versus the sandbox overhead.\n - a kubernetes service provider can subsidize the charge-back model potentially and eat the cost of the\n runtime choice, but charge the user for the cpu/memory consumed independent of runtime choice.\n\n\n### Leaving the PodSpec unchanged\n\nInstead of tracking the overhead associated with running a workload with a given runtimeClass in the PodSpec,\nthe Kubelet (for pod cgroup creation), the scheduler (for honoring reqests overhead for the pod) and the resource\nquota handling (for optionally taking requests overhead of a workload into account) will need to be augmented\nto add a sandbox overhead when applicable.\n\nPros:\n * no changes to the pod spec\n * user does not have the option of setting the overhead\n * no need for a mutating admission controller\n\nCons:\n * handling of the pod overhead is spread out across a few components\n * Not user perceptible from a workload perspective.\n * very complicated if the runtimeClass policy changes after workloads are running\n"
+  },
+  {
+    "id": "89008358b6e464eb1bcc9933c4146d58",
+    "title": "Shared PID Namespace",
+    "authors": ["@verb"],
+    "owningSig": "sig-node",
+    "participatingSigs": ["sig-node"],
+    "reviewers": ["@yujuhong"],
+    "approvers": ["@dchen1107"],
+    "editor": "N/A",
+    "creationDate": "2016-12-21",
+    "lastUpdated": "2019-09-30",
+    "status": "implemented",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Pod Shared PID Namespace\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n  - [Implementation Details](#implementation-details)\n    - [Kubernetes API Changes](#kubernetes-api-changes)\n    - [Container Runtime Interface Changes](#container-runtime-interface-changes)\n      - [Targeting a Specific Container's Namespace](#targeting-a-specific-containers-namespace)\n    - [dockershim Changes](#dockershim-changes)\n    - [Deprecation of existing kubelet flag](#deprecation-of-existing-kubelet-flag)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n  - [Explicit Container/Sandbox ID Targeting](#explicit-containersandbox-id-targeting)\n  - [Defaulting to PID Namespace Sharing](#defaulting-to-pid-namespace-sharing)\n  - [Migration to Shared-only Namespaces](#migration-to-shared-only-namespaces)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [x] KEP approvers have set the KEP status to `implementable`\n- [x] Design details are appropriately documented\n- [x] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [x] Graduation criteria is in place\n- [x] \"Implementation History\" section is up-to-date for milestone\n- [x] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [x] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nWe will add support for configuring pod-shared process namespaces by adding a\nnew boolean field `ShareProcessNamespace` to the pod spec. The default to false\nmeans that each container will have a separate process namespace. When set to\ntrue, all containers in the pod will share a single process namespace.\n\nThe Container Runtime Interface (CRI) will be updated to support three namespace\nmodes: Container, Pod \u0026 Node. The Runtime Manager will translate the pod spec\ninto one of these modes as follows:\n\nPod `shareProcessNamespace` | Pod `hostPID` | CRI PID Mode\nfalse                       | false         | Container\nfalse                       | true          | Node\ntrue                        | false         | Pod\ntrue                        | true          | *Error*\n\nIf a runtime does not implement a particular PID mode, it must return an error.\nFor reference, Docker will support all three modes when using version \u003e= 1.13.1.\n\nThe shared PID functionality will be hidden behind a new feature gate in both\nthe API server and the kubelet, and the existing `--docker-disable-shared-pid`\nflag will be removed from the kubelet, subject to\n[deprecation policy](https://kubernetes.io/docs/reference/deprecation-policy/).\n\n## Motivation\n\nPods share namespaces where possible, but support for sharing the PID namespace\nhad not been defined due to lack of support in Docker. This created an implicit\nAPI on which certain container images now rely. This document proposes adding\nsupport for sharing a process namespace between containers in a pod while\nmaintaining backwards compatibility with the existing implicit API.\n\n### Goals\n\n*   Backwards compatibility with container images expecting `pid == 1` semantics\n*   Per-pod configuration of PID namespace sharing\n*   Ability to change default sharing behavior in `v2.Pod`\n\n### Non-Goals\n\n*   Creating a general purpose container init solution\n*   Multiple shared PID namespaces per pod\n*   Per-container configuration of PID namespace sharing\n\n## Proposal\n\n### User Stories\n\nSharing a PID namespace between containers in a pod is discussed in\n[#1615](https://issues.k8s.io/1615) and enables:\n\n1.  signaling between containers, which is useful for side cars (e.g. for\n    signaling a daemon process after rotating logs).\n1.  easier troubleshooting of pods.\n1.  addressing [Docker's zombie problem][docker-zombies] by reaping orphaned\n    zombies in the infra container.\n\n[docker-zombies]: https://blog.phusion.nl/2015/01/20/docker-and-the-pid-1-zombie-reaping-problem/\n\n### Implementation Details\n\n#### Kubernetes API Changes\n\n`v1.PodSpec` gains a new field named `ShareProcessNamespace`:\n\n```\n// PodSpec is a description of a pod.\ntype PodSpec struct {\n    ...\n       // Use the host's pid namespace.\n       // Note that HostPID and ShareProcessNamespace cannot both be set.\n       // Optional: Default to false.\n       // +k8s:conversion-gen=false\n       // +optional\n       HostPID bool `json:\"hostPID,omitempty\" protobuf:\"varint,12,opt,name=hostPID\"`\n       // Share a single process namespace between all of the containers in a pod.\n       // Note that HostPID and ShareProcessNamespace cannot both be set.\n       // Optional: Default to false.\n       // +k8s:conversion-gen=false\n       // +optional\n       ShareProcessNamespace *bool `json:\"shareProcessNamespace,omitempty\" protobuf:\"varint,XX,opt,name=shareProcessNamespace\"`\n       ...\n```\n\nThe field name deviates from that of HostPID in an attempt to\n[better signal the consequences](https://github.com/kubernetes/community/pull/1048/files#r159146536)\nof setting the option. Setting both `ShareProcessNamespace` and `HostPID` will\ncause a validation error.\n\n#### Container Runtime Interface Changes\n\nNamespace options in the CRI are currently specified for both `PodSandbox` and\n`Container` creation requests via booleans in `NamespaceOption`:\n\n```\nmessage NamespaceOption {\n    // If set, use the host's network namespace.\n     bool host_network = 1;\n     // If set, use the host's PID namespace.\n     bool host_pid = 2;\n     // If set, use the host's IPC namespace.\n     bool host_ipc = 3;\n}\n```\n\nWe will change `NamespaceOption` to use a `NamespaceMode` enumeration for the\nexisting namespace options:\n\n```\nenum NamespaceMode {\n    POD       = 0;\n    CONTAINER = 1;\n    NODE      = 2;\n}\n\n// NamespaceOption provides options for Linux namespaces.\nmessage NamespaceOption {\n    // Network namespace for this container/sandbox.\n    // Runtimes must support: POD, NODE\n    NamespaceMode network = 1;\n    // PID namespace for this container/sandbox.\n    // Note: The CRI default is POD, but the v1.PodSpec default is CONTAINER.\n    // The kubelet's runtime manager will set this to CONTAINER explicitly for v1 pods.\n    // Runtimes must support: POD, CONTAINER, NODE\n    NamespaceMode pid = 2;\n    // IPC namespace for this container/sandbox.\n    // Runtimes must support: POD, NODE\n    NamespaceMode ipc = 3;\n}\n```\n\nNote that this breaks backwards compatibility in the CRI, which is still in\nalpha.\n\nThe protocol default for a namespace is `POD` because that's the default for\nnetwork and IPC, and we will consider making it the default for PID in `v2.Pod`.\nThe kubelet will explicitly set `pid` to `CONTAINER` for `v1.Pod` by default so\nthat the default behavior of `v1.Pod` does not change.\n\nThis CRI design allows different namespace configuration for each of the\ncontainers in the pod and the sandbox, but currently we have no plans to support\nthis in the Kubernetes API. The kubelet will translate namespace booleans from\nv1.PodSpec into a single `NamespaceMode` to be used for the sandbox and all\nregular and init containers in a pod.\n\n##### Targeting a Specific Container's Namespace\n\nThough we don't intend to support this in general pod configuration, there is a\nuse case for mixed process namespaces within a single pod. [Ephemeral\nContainers](https://features.k8s.io/277) allows inserting an ephemeral Debug\nContainer in an existing, running pod. In order for this to be useful we want\nto share, within the pod, a process namespace between the new container\nperforming the debugging and its existing target container.\n\nThis is done with the additional `NamespaceMode` `TARGET` and field `target_id`:\n\n```\nenum NamespaceMode {\n    POD       = 0;\n    CONTAINER = 1;\n    NODE      = 2;\n    TARGET    = 3;\n}\n\n// NamespaceOption provides options for Linux namespaces.\nmessage NamespaceOption {\n    // Network namespace for this container/sandbox.\n    // Runtimes must support: POD, NODE\n    NamespaceMode network = 1;\n    // PID namespace for this container/sandbox.\n    // Note: The CRI default is POD, but the v1.PodSpec default is CONTAINER.\n    // The kubelet's runtime manager will set this to CONTAINER explicitly for v1 pods.\n    // Runtimes must support: POD, CONTAINER, NODE, TARGET\n    NamespaceMode pid = 2;\n    // IPC namespace for this container/sandbox.\n    // Runtimes must support: POD, NODE\n    NamespaceMode ipc = 3;\n    // Target Container ID for NamespaceMode of TARGET. This container must be in the\n    // same pod as the target container.\n    string target_id = 4;\n}\n```\n\nWhen `NamespaceOption.pid` is set to `TARGET`, a runtime must create the new\ncontainer in the namespace used by the container ID in `target_id`. If the\ntarget container has `NamespaceOption.pid` set to `POD`, then the new container\nshould also use the pod namespace. If the target container has an isolated\nprocess namespace, then the new container will join only that container's\nnamespace. Examples are provided for dockershim below.\n\nThere is no mechanism in the Kubernetes API for an end-user to set `TARGET`. It\nexists for the kubelet to run automation or debugging from a container image in\nthe namespace of an existing pod and container. Additionally, we choose to\nexplicitly not support sharing namespaces between different pods. The kubelet\nmust not generate such a reference, and the runtime should not accept it. That\nis, for pod{Container `A`, Container `B`, Sandbox `S}` and any other unrelated\nContainer `C`:\n\nvalid `target_id` | invalid `target_id`\ncontainerID(A)    | sandboxID(S)\ncontainerID(B)    | containerID(C)\n\n#### dockershim Changes\n\nThe Docker runtime implements the pod sandbox as a container running the pause\ncontainer image. When configured for `POD` namespace sharing, the PID namespace\nof the sandbox will become the single PID namespace for the pod. This means a\nnamespace of `POD` and `CONTAINER` are equivalent for the sandbox. The mapping\nof the _sandbox's_ PID mode to docker's `HostConfig.PidMode` is (`v1.Pod`\nsettings provided as reference):\n\nShareProcessNamespace | HostPID | Sandbox PID Mode | HostConfig.PidMode\nfalse                 | false   | CONTAINER        | *unset*\ntrue                  | false   | POD              | *unset*\nfalse                 | true    | NODE             | \"host\"\n\\-                    | \\-      | TARGET           | *Error*\n\nFor _containers_, `HostConfig.PidMode` will be set as follows:\n\nShareProcessNamespace | HostPID | Container PID Mode | HostConfig.PidMode\nfalse                 | false   | CONTAINER          | *unset*\ntrue                  | false   | POD                | \"container:[sandbox-container-id]\"\nfalse                 | true    | NODE               | \"host\"\nfalse                 | false   | TARGET             | \"container:[target-container-id]\"\ntrue                  | false   | TARGET             | \"container:[sandbox-container-id]\"\nfalse                 | true    | TARGET             | \"host\"\n\nIf the Docker runtime version does not support sharing pid namespaces, a\n`CreateContainerRequest` with `namespace_options.pid` set to `POD` will return\nan error.\n\n#### Deprecation of existing kubelet flag\n\nSIG Node did not anticipate the strong objections to migrating from isolated to\nshared process namespaces for Docker. The previous (now abandoned) migration\nplan introduced a kubelet flag to toggle the shared namespace behavior, but\nobjections did not materialize until the flag had moved from experimental to GA.\n\nThe `--docker-disable-shared-pid` (default: true) kubelet flag disables the use\nof shared process namespaces for the Docker runtime. We will immediately mark it\nas deprecated, but according to the\n[deprecation policy](https://kubernetes.io/docs/reference/deprecation-policy/)\nwe must support it for 6 months.\n\nWe must provide a transition path for users setting this kubelet flag to false.\nSetting this flag asserts a desire to override the default Kubernetes behavior\nfor all pods. Until the flag is removed, the kubelet will honor this assertion\nby ignoring the value of `ShareProcessNamespace` and logging a warning to the\nevent log.\n\n### Risks and Mitigations\n\nSharing a process namespace fits well with Kubernetes' pod abstraction, but it's\na significant departure from the traditional behavior of Docker. This may break\ncontainer images and development patterns that have come to rely on process\nisolation. Notably:\n\n1.  **The main container process no longer has PID 1**. It cannot be signalled\n    using `kill 1`, and attempting to do so will instead signal the\n    infrastructure container and potentially restart the pod. Containers\n    shipping an init system like systemd may\n    [require additional flags](https://github.com/kubernetes/kubernetes/issues/48937#issuecomment-321243669).\n1.  **Processes are visible to other containers in the pod**. This includes all\n    information visible in `/proc`, such as passwords as arguments or\n    environment variables, and process signalling. This can be somewhat\n    mitigated by running processes as separate, non-root users.\n1.  **Container filesystems are visible to other containers in the pod through\n    the \u003ccode\u003e/proc/$pid/root\u003c/code\u003e magic symlink**. This makes debugging\n    easier, but it also means that secrets are protected only by standard\n    filesystem permissions.\n\n## Design Details\n\n### Test Plan\n\nThis feature launched with test coverage in node-e2e.\n\n### Graduation Criteria\n\nAt the time this KEP was written, the feature was already in beta.\n\n#### Beta -\u003e GA Graduation\n\n- 3 articles describing using `shareProcessNamespace` in a task.\n- `shareProcessNamespace` referenced in documentation for a Kubernetes cloud\n  provider (as proxy indicator for feature enabled on cloud provider).\n- Feature spends at least 2 release in beta.\n- No open bug reports for latest release.\n\n## Implementation History\n\n- 2016-12-21: [Original (pre-KEP) proposal created][original-proposal]\n- 1.10: Feature released in Alpha.\n- 1.12: Feature released in Beta.\n- 2019-09-20: Ported [Original proposal][original-proposal] to KEP.\n- 1.17: Feature generally available.\n\n[original-proposal]: https://git.k8s.io/community/contributors/design-proposals/node/pod-pid-namespace.md\n\n## Alternatives\n\n### Explicit Container/Sandbox ID Targeting\n\nRather than using a `NamespaceMode`, `NamespaceOption.pid` could be a string\nthat explicitly targets a container or sandbox ID:\n\n```\n// NamespaceOption provides options for Linux namespaces.\nmessage NamespaceOption {\n    ...\n    // ID of Sandbox or Container to use for PID namespace, or \"host\"\n    string pid = 2;\n    ...\n}\n```\n\nThis removes the need for a separate `TARGET` mode, but a mode enumeration\nbetter captures the intent of the option.\n\n### Defaulting to PID Namespace Sharing\n\nOther Kubernetes runtimes already share a single PID namespace between\ncontainers in a pod. We could easily change the Docker runtime to always share a\nPID namespace when supported by the installed Docker version, but this would\ncause problems for container images that assume they will always be PID 1.\n\n### Migration to Shared-only Namespaces\n\nRather than adding support to the API for configuring namespaces we could allow\nchanging the default behavior with pod annotations with the intention of\nremoving support for isolated PID namespaces in v2.Pod. Many members of the\ncommunity want to use the isolated namespaces as security boundary between\ncontainers in a pod, however.\n"
+  },
+  {
+    "id": "bdb750e28879613d1961ce5af9d8100a",
+    "title": "Kubelet endpoint for device assignment observation details",
+    "authors": ["@dashpole", "@vikaschoudhary16"],
+    "owningSig": "sig-node",
+    "participatingSigs": null,
+    "reviewers": ["@thockin", "@derekwaynecarr", "@dchen1107", "@vishh"],
+    "approvers": ["@sig-node-leads"],
+    "editor": "@dashpole",
+    "creationDate": "2018-07-19",
+    "lastUpdated": "2019-04-30",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "# Kubelet endpoint for device assignment observation details \n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Abstract](#abstract)\n- [Background](#background)\n- [Objectives](#objectives)\n- [User Journeys](#user-journeys)\n  - [Device Monitoring Agents](#device-monitoring-agents)\n- [Changes](#changes)\n  - [Potential Future Improvements](#potential-future-improvements)\n- [Alternatives Considered](#alternatives-considered)\n  - [Add v1alpha1 Kubelet GRPC service, at \u003ccode\u003e/var/lib/kubelet/pod-resources/kubelet.sock\u003c/code\u003e, which returns a list of \u003ca href=\"https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/apis/cri/runtime/v1alpha2/api.proto#L734\"\u003eCreateContainerRequest\u003c/a\u003es used to create containers.](#add-v1alpha1-kubelet-grpc-service-at--which-returns-a-list-of-createcontainerrequests-used-to-create-containers)\n  - [Add a field to Pod Status.](#add-a-field-to-pod-status)\n  - [Use the Kubelet Device Manager Checkpoint file](#use-the-kubelet-device-manager-checkpoint-file)\n  - [Add a field to the Pod Spec:](#add-a-field-to-the-pod-spec)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Abstract\nIn this document we will discuss the motivation and code changes required for introducing a kubelet endpoint to expose device to container bindings.\n\n## Background\n[Device Monitoring](https://docs.google.com/document/d/1NYnqw-HDQ6Y3L_mk85Q3wkxDtGNWTxpsedsgw4NgWpg/edit?usp=sharing) requires external agents to be able to determine the set of devices in-use by containers and attach pod and container metadata for these devices.\n\n## Objectives\n\n* To remove current device-specific knowledge from the kubelet, such as [accellerator metrics](https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/apis/stats/v1alpha1/types.go#L229)\n* To enable future use-cases requiring device-specific knowledge to be out-of-tree\n\n## User Journeys\n\n### Device Monitoring Agents\n\n* As a _Cluster Administrator_, I provide a set of devices from various vendors in my cluster.  Each vendor independently maintains their own agent, so I run monitoring agents only for devices I provide.  Each agent adheres to to the [node monitoring guidelines](https://docs.google.com/document/d/1_CdNWIjPBqVDMvu82aJICQsSCbh2BR-y9a8uXjQm4TI/edit?usp=sharing), so I can use a compatible monitoring pipeline to collect and analyze metrics from a variety of agents, even though they are maintained by different vendors.\n* As a _Device Vendor_, I manufacture devices and I have deep domain expertise in how to run and monitor them. Because I maintain my own Device Plugin implementation, as well as Device Monitoring Agent, I can provide consumers of my devices an easy way to consume and monitor my devices without requiring open-source contributions. The Device Monitoring Agent doesn't have any dependencies on the Device Plugin, so I can decouple monitoring from device lifecycle management. My Device Monitoring Agent works by periodically querying the `/devices/\u003cResourceName\u003e` endpoint to discover which devices are being used, and to get the container/pod metadata associated with the metrics:\n\n![device monitoring architecture](https://user-images.githubusercontent.com/3262098/43926483-44331496-9bdf-11e8-82a0-14b47583b103.png)\n\n\n## Changes\n\nAdd a v1alpha1 Kubelet GRPC service, at `/var/lib/kubelet/pod-resources/kubelet.sock`, which returns information about the kubelet's assignment of devices to containers. It obtains this information from the internal state of the kubelet's Device Manager. The GRPC Service returns a single PodResourcesResponse, which is shown in proto below:\n```protobuf\n// PodResources is a service provided by the kubelet that provides information about the\n// node resources consumed by pods and containers on the node\nservice PodResources {\n    rpc List(ListPodResourcesRequest) returns (ListPodResourcesResponse) {}\n}\n\n// ListPodResourcesRequest is the request made to the PodResources service\nmessage ListPodResourcesRequest {}\n\n// ListPodResourcesResponse is the response returned by List function\nmessage ListPodResourcesResponse {\n    repeated PodResources pod_resources = 1;\n}\n\n// PodResources contains information about the node resources assigned to a pod\nmessage PodResources {\n    string name = 1;\n    string namespace = 2;\n    repeated ContainerResources containers = 3;\n}\n\n// ContainerResources contains information about the resources assigned to a container\nmessage ContainerResources {\n    string name = 1;\n    repeated ContainerDevices devices = 2;\n}\n\n// ContainerDevices contains information about the devices assigned to a container\nmessage ContainerDevices {\n    string resource_name = 1;\n    repeated string device_ids = 2;\n}\n```\n\n### Potential Future Improvements\n\n* Add `ListAndWatch()` function to the GRPC endpoint so monitoring agents don't need to poll.\n* Add identifiers for other resources used by pods to the `PodResources` message.\n  * For example, persistent volume location on disk\n\n## Alternatives Considered\n\n### Add v1alpha1 Kubelet GRPC service, at `/var/lib/kubelet/pod-resources/kubelet.sock`, which returns a list of [CreateContainerRequest](https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/apis/cri/runtime/v1alpha2/api.proto#L734)s used to create containers.\n* Pros:\n  * Reuse an existing API for describing containers rather than inventing a new one\n* Cons:\n  * It ties the endpoint to the CreateContainerRequest, and may prevent us from adding other information we want in the future\n  * It does not contain any additional information that will be useful to monitoring agents other than device, and contains lots of irrelevant information for this use-case.\n* Notes:\n  * Does not include any reference to resource names.  Monitoring agentes must identify devices by the device or environment variables passed to the pod or container.\n\n### Add a field to Pod Status. \n* Pros:\n  * Allows for observation of container to device bindings local to the node through the `/pods` endpoint\n* Cons:\n  * Only consumed locally, which doesn't justify an API change\n  * Device Bindings are immutable after allocation, and are _debatably_ observable (they can be \"observed\" from the local checkpoint file).  Device bindings are generally a poor fit for status.\n\n### Use the Kubelet Device Manager Checkpoint file\n* Allows for observability of device to container bindings through what exists in the checkpoint file\n  * Requires adding additional metadata to the checkpoint file as required by the monitoring agent\n* Requires implementing versioning for the checkpoint file, and handling version skew between readers and the kubelet\n* Future modifications to the checkpoint file are more difficult.\n\n### Add a field to the Pod Spec:\n* A new object `ComputeDevice` will be defined and a new variable `ComputeDevices` will be added in the `Container` (Spec) object which will represent a list of `ComputeDevice` objects.\n```golang\n// ComputeDevice describes the devices assigned to this container for a given ResourceName\ntype ComputeDevice struct {\n\t// DeviceIDs is the list of devices assigned to this container\n\tDeviceIDs []string\n\t// ResourceName is the name of the compute resource\n\tResourceName string\n}\n\n// Container represents a single container that is expected to be run on the host.\ntype Container struct {\n    ...\n\t// ComputeDevices contains the devices assigned to this container\n\t// This field is alpha-level and is only honored by servers that enable the ComputeDevices feature.\n\t// +optional\n\tComputeDevices []ComputeDevice\n\t...\n}\n```\n* During Kubelet pod admission, if `ComputeDevices` is found non-empty, specified devices will be allocated otherwise behaviour will remain same as it is today.\n* Before starting the pod, the kubelet writes the assigned `ComputeDevices` back to the pod spec.  \n  * Note: Writing to the Api Server and waiting to observe the updated pod spec in the kubelet's pod watch may add significant latency to pod startup.\n* Allows devices to potentially be assigned by a custom scheduler.\n* Serves as a permanent record of device assignments for the kubelet, and eliminates the need for the kubelet to maintain this state locally.\n\n## Graduation Criteria\n\nAlpha:\n- [x] Implement the endpoint as described above\n- [x] E2e node test tests the endpoint: https://k8s-testgrid.appspot.com/sig-node-kubelet#node-kubelet-serial\u0026include-filter-by-regex=DevicePluginProbe\n\nBeta:\n- [x] Demonstrate in production environments that the endpoint can be used to replace in-tree GPU device metrics (NVIDIA, sig-node April 30, 2019).\n\n## Implementation History\n\n- 2018-09-11: Final version of KEP (proposing pod-resources endpoint) published and presented to sig-node.  [Slides](https://docs.google.com/presentation/u/1/d/1xz-iHs8Ec6PqtZGzsmG1e68aLGCX576j_WRptd2114g/edit?usp=sharing)\n- 2018-10-30: Demo with example gpu monitoring daemonset\n- 2018-11-10: KEP lgtm'd and approved\n- 2018-11-15: Implementation and e2e test merged before 1.13 release: kubernetes/kubernetes#70508\n- 2019-04-30: Demo of production GPU monitoring by NVIDIA\n- 2019-04-30: Agreement in sig-node to move feature to beta in 1.15\n"
+  },
+  {
+    "id": "a6de75d59a485c6113c2df6597a4aad4",
+    "title": "Kubelet Resource Metrics Endpoint",
+    "authors": ["@dashpole"],
+    "owningSig": "sig-node",
+    "participatingSigs": ["sig-instrumentation"],
+    "reviewers": ["DirectXMan12", "tallclair"],
+    "approvers": ["dchen1107", "brancz"],
+    "editor": "",
+    "creationDate": "2019-01-24",
+    "lastUpdated": "2019-02-21",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Kubelet Resource Metrics Endpoint\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Background](#background)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [API](#api)\n- [Future Improvements](#future-improvements)\n- [Benchmarking](#benchmarking)\n  - [Round 1](#round-1)\n    - [Methods](#methods)\n    - [Results](#results)\n  - [Round 2](#round-2)\n    - [Methods](#methods-1)\n    - [Results](#results-1)\n- [Alternatives Considered](#alternatives-considered)\n  - [gRPC API](#grpc-api)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThe Kubelet Resource Metrics Endpoint is a new kubelet metrics endpoint which serves metrics required by the cluster-level [Resource Metrics API](https://github.com/kubernetes/metrics#resource-metrics-api).  The proposed design uses the prometheus text format, and provides the minimum required metrics for serving the [Resource Metrics API](https://github.com/kubernetes/metrics#resource-metrics-api).\n\n## Background\n\nThe [Monitoring Architecture](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/monitoring_architecture.md) proposal established separate pipelines for Resource Metrics, and for Monitoring Metrics.  The [Core Metrics](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/core-metrics-pipeline.md#core-metrics-in-kubelet) proposal describes the set of metrics that we consider core, and their uses.  Note that the term “core” is overloaded, and this document will refer to these as Resource Metrics, since they are for first class kubernetes resources and are served by the [Resource Metrics API](https://github.com/kubernetes/metrics#resource-metrics-api) at the cluster-level.\n\nA [previous proposal](https://docs.google.com/document/d/1_CdNWIjPBqVDMvu82aJICQsSCbh2BR-y9a8uXjQm4TI/edit?usp=sharing) by @DirectXMan12 also proposed a prometheus endpoint.  The [kubernetes metrics overhaul KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/0031-kubernetes-metrics-overhaul.md#export-less-metrics) acknowledges the need to export fewer metrics from the kubelet.  This new API is a step in that direction, as it eliminates the Metric Server's dependency on the Summary API.\n\nFor the purposes of this document, I will use the following definitions:\n\n* Resource Metrics: Metrics for the consumption of first-class resources (CPU, Memory, Ephemeral Storage) which are aggregated by the [Metrics Server](https://github.com/kubernetes-incubator/metrics-server#kubernetes-metrics-server), and served by the [Resource Metrics API](https://github.com/kubernetes/metrics#resource-metrics-api)\n* Monitoring Metrics: Metrics for observability and introspection of the cluster, which are used by end-users, operators, devs, etc. \n\n\nThe Kubelet’s [JSON Summary API](https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/apis/stats/v1alpha1/types.go) is currently used by the [Metrics Server](https://github.com/kubernetes-incubator/metrics-server#kubernetes-metrics-server).  It contains far more metrics than are required by the Metrics Server.\n\n[Prometheus](https://prometheus.io/) is commonly used for exposing metrics for kubernetes components, and the [Prometheus Operator](https://github.com/coreos/prometheus-operator#prometheus-operator), which Sig-Instrumentation works on, is commonly used to deploy and manage metrics collection.\n\n[OpenMetrics](https://openmetrics.io/) is a new prometheus-based metric standard which supports both text and protobuf.  \n\n[GRPC](https://grpc.io/) is commonly used for interfaces between components in kubernetes, such as the [Container Runtime Interface](https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/apis/cri/runtime/v1alpha2/api.proto).  GRPC uses [protocol-buffers](https://developers.google.com/protocol-buffers/docs/overview) (protobuf) for serialization and deserialization, which is more performant than other formats.\n\n## Motivation\n\nThe Kubelet Summary API is a source of both Resource and Monitoring Metrics.  Because of it’s dual purpose, it does a poor job of both.  It provides much more information than required by the Metrics Server, as demonstrated by [kubernetes/kubernetes#68841](https://github.com/kubernetes/kubernetes/pull/68841).  Additionally, we have pushed back on adding metrics to the Summary API for monitoring, such as DiskIO or tcp/udp metrics, because they are expensive to collect, and not required by all users.\n\nThis proposal deals with the first problem, which is that the Summary API is a poor provider of Resource Metrics.  It proposes a purpose-built API for supplying Resource Metrics.\n\n### Goals\n\n* [Primary] Provide the minimum set of metrics required to serve the Resource Metrics API\n* [Secondary] Minimize the CPU and Memory footprint of the metrics server due to collecting metrics\n  * Perform efficiently at frequent (sub-second) rates of metrics collection\n* [Secondary] Use a format that is familiar to the kubernetes community, which can be consumed by common monitoring pipelines, and is interoperable with commonly-used monitoring pipelines.\n\n### Non-Goals\n\n* Deprecate or remove the Summary API\n* Add new Resource Metrics to the metrics server (e.g. Ephemeral Storage)\n* Detail how the kubelet will collect metrics to support this API.\n* Determine what the pipeline for “Monitoring” metrics will look like\n\n## Proposal\n\nThe kubelet will expose an endpoint at `/metrics/resource/v1alpha1` in prometheus text exposition format using the prometheus client library.\n\n\n### API\n\n```\n# Cumulative cpu time consumed by a container in seconds\nName: container_cpu_usage_seconds_total\nLabels: container, pod, namespace\n\n# Current working set of a container in bytes\nName: container_memory_working_set_bytes\nLabels: container, pod, namespace\n\n# Cumulative cpu time consumed by the node in seconds\nName: node_cpu_usage_seconds_total\nLabels: \n\n# Current working set of the node in bytes\nName: node_memory_working_set_bytes\nLabels:\n```\n\nExplicit timestamps (see the [prometheus exposition format docs](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#comments-help-text-and-type-information)) will be added to metrics because metrics are (currently) collected out-of-band and cached.  We make no guarantees about the age of metrics, but include the timestamp to allow readers to correctly calculate rates, etc.  Timestamps are currently required because metrics are collected out-of-band by cAdvisor.  This deviates from the [prometheus best practices](https://prometheus.io/docs/instrumenting/writing_exporters/#scheduling), and we should attempt to migrate to synchronous collection during each scrape in the future.\n\nUse separate metrics for node and containers to avoid “magic” container names, such as “machine”.\n\nCurrently the Metrics Server uses a 10s average of CPU usage provided by the kubelet summary API.  The kubelet should provide the raw cumulative CPU usage so the metrics server can determine the time period over which it wants to take the rate.\n\nLabels are named in accordance with the [kubernetes instrumentation guidelines](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/instrumentation.md#resource-referencing), and thus are named `pod`, rather than `pod_name`.\n\nExample implementation: https://github.com/kubernetes/kubernetes/compare/master...dashpole:prometheus_core_metrics\n\n## Future Improvements\n\n[OpenMetrics](https://openmetrics.io/) is an upcoming prometheus-based standard which has support for protocol buffers.  By using this format when it becomes available, we can further improve the efficiency of the Resource Metrics Pipeline, while maintaining compatibility with other monitoring pipelines.\n\n## Benchmarking\n\n### Round 1\n\nThis experiment compares the current JSON Summary API to prometheus and GRPC at 1s and 30s scrape intervals.  Prometheus uses basic text parsing, and grpc uses a basic `Get()` API.\n\n#### Methods\n\nThe setup has 10 nodes, 500 pods, and 6500 containers (running pause).  Nodes have 1 CPU core, and 3.75Gb memory.  The same cluster was used for all benchmarks for consistency, with a different Metrics Server running.  The values below are the maximum values reported during a 10 minute period.\n\n#### Results\n\nWe can see that GRPC has the lowest CPU usage of all formats tested, and is an order-of-magnitude improvement over the current JSON Summary API.  Memory Usage for both GRPC and Prometheus are similarly lower than the JSON Summary API.\n\n\u003cimg src=\"https://user-images.githubusercontent.com/3262098/51704936-1dca4f80-1fcf-11e9-9485-b4c765a5a1c9.png\" width=\"600\" height=\"375\"\u003e\n\n\u003cimg src=\"https://user-images.githubusercontent.com/3262098/51704931-1acf5f00-1fcf-11e9-93aa-8004b43e6770.png\" width=\"600\" height=\"375\"\u003e\n\n### Round 2\n\nAfter learning that the prometheus server achieves better performance with caching, I performed an additional round of tests.  These used a metrics-server which caches metric descriptors it has parsed before, and tested with larger numbers of container metrics.\n\nThis experiment compares basic prometheus, optimized prometheus parsing and GRPC at 1s scrape intervals with higher numbers of container metrics.  \"Unoptimized Prometheus\" uses basic text parsing, \"Prometheus w/ Caching\" borrows [caching logic from the prometheus server](https://github.com/prometheus/prometheus/blob/master/scrape/scrape.go#L991) to avoid re-parsing metric descriptors it has already parsed and grpc uses a basic `Get()` API.\n\n#### Methods\n\nThe setup has 10 nodes, and up to 40,000 containers (running pause).  Nodes have 2 CPU core, and 7.5Gb memory.  The same cluster was used for all benchmarks for consistency, with a different Metrics Server running.  The values below are the maximum values reported during a 10 minute period.\n\nThis experiment \"fakes\" large numbers of containers by having the kubelet return 100 container metrics for each actual container run on the node.\n\n#### Results\n\nBoth gRPC and the optimized prometheus were able to scale to 40k containers.  The gRPC implementation was more efficient by a factor of approx. 3.  \n\n\u003cimg src=\"https://user-images.githubusercontent.com/3262098/51704923-173bd800-1fcf-11e9-910d-3fd6606550f3.png\" width=\"600\" height=\"375\"\u003e\n\n\u003cimg src=\"https://user-images.githubusercontent.com/3262098/51704880-02f7db00-1fcf-11e9-8034-c64f971a2204.png\" width=\"600\" height=\"375\"\u003e\n\n## Alternatives Considered\n\n### gRPC API\n\nAs demonstrated in the benchmarks above, the proto-based gRPC endpoint is the most efficient in terms of CPU and Memory usage.  Such an endpoint could potentially be improved by using streaming, rather than scraping to be even more efficient at high rates of collection.\n\nHowever, given the prevalence of the Prometheus format within the kubernetes community, gRPC is not as compatible with common monitoring pipelines.  The endpoint would _only_ be useful for supplying metrics for the Metrics Server, or monitoring components that integrate directly with it.\n\nWhen using caching in the Metrics Server, the prometheus text format performs _well enough_ for us to prefer prometheus over gRPC given the prevalence of prometheus in the community.  When the OpenMetrics format becomes stable, we can get even closer to the performance of gRPC by using the proto-based format.\n\n```\n// Usage is a set of resources consumed\nmessage Usage {\n  int64 time = 1;\n  uint64 cpu_usage_core_nanoseconds_total = 2;\n  uint64 memory_working_set_bytes = 3;\n}\n// ContainerUsage is the resource usage for a single container\nmessage ContainerUsage {\n  string name = 1;\n  Usage usage = 2;\n}\n// PodUsage is the resource usage for a pod\nmessage PodUsage {\n  string name = 1;\n  string namespace = 2;\n  repeated ContainerUsage containers = 3;\n}\n// MetricsResponse is sent by plugin to kubelet in response to MetricsRequest RPC\nmessage MetricsResponse {\n  Usage node = 1;\n  repeated PodUsage pods = 2;\n}\n// MetricsRequest is the empty request message for Kubelet\nmessage MetricsRequest {}\n// ResourceMetrics is the service advertised by the kubelet for usage metrics.\nservice ResourceMetrics {\n  rpc Get(MetricsRequest) returns (MetricsResponse) {}\n}\n```\n\n## Graduation Criteria\n\nAlpha:\n\n- [ ] Implement the kubelet resource metrics endpoint as described above\n- [ ] Test the new endpoint with a node-e2e test similar to the current summary API test\n- [ ] Modify the metrics server to consume the kubelet resource metrics endpoint 3 releases after it is added to the kubelet\n\nBeta/GA:\n\n- [ ] Determine whether a transition to OpenMetrics format is required, and make those changes if necessary\n- [ ] Add node-e2e test to the node conformance tests\n\n## Implementation History\n\n- 2019-01-24: Initial KEP published.\n- 2019-01-29: Presentation to Sig-Node\n- 2019-02-04: KEP gets LGTM and Approval\n- 2019-02-07: Presentation to Sig-Instrumentation\n"
+  },
+  {
+    "id": "f5504084a854ad05a476e40d2dcf4a3a",
+    "title": "RuntimeClass Scheduling",
+    "authors": ["@tallclair"],
+    "owningSig": "sig-node",
+    "participatingSigs": ["sig-scheduling"],
+    "reviewers": ["yastij", "egernst"],
+    "approvers": ["bsalamat", "dchen1107", "derekwaynecarr"],
+    "editor": "",
+    "creationDate": "2019-03-14",
+    "lastUpdated": "2019-05-20",
+    "status": "implementable",
+    "seeAlso": ["/keps/sig-node/runtime-class.md"],
+    "replaces": [
+      "[RuntimeClass Scheduling Brainstorm](https://docs.google.com/document/d/1W51yBNTvp0taeEss56GTk8jczqFJ2d6jBeN6sCSlYZU/edit#)"
+    ],
+    "supersededBy": null,
+    "markdown": "\n# RuntimeClass Scheduling\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Windows](#windows)\n    - [Sandboxed Nodes](#sandboxed-nodes)\n- [Design Details](#design-details)\n  - [API](#api)\n  - [RuntimeClass Admission Controller](#runtimeclass-admission-controller)\n  - [Labeling Nodes](#labeling-nodes)\n  - [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n  - [Scheduler](#scheduler)\n  - [RuntimeController Mix-in](#runtimecontroller-mix-in)\n    - [RuntimeController](#runtimecontroller)\n    - [Mix-in](#mix-in)\n  - [NodeSelector](#nodeselector)\n  - [Native RuntimeClass Reporting](#native-runtimeclass-reporting)\n  - [Scheduling Policy](#scheduling-policy)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nRuntimeClass scheduling enables native support for heterogeneous clusters where\nevery node does not necessarily support every RuntimeClass. This feature allows\npod authors to select a RuntimeClass without needing to worry about cluster\ntopology.\n\n## Motivation\n\nIn the initial RuntimeClass implementation, we explicitly assumed that the\ncluster nodes were homogenous with regards to RuntimeClasses. It was still\npossible to run a heterogeneous cluster, but pod authors would need to set\nappropriate [NodeSelector][] rules and [tolerations][taint-and-toleration] to\nensure the pods landed on supporting nodes.\n\nAs [use cases](#user-stories) have appeared and solidified, it has become clear\nthat heterogeneous clusters will not be uncommmon, and supporting a smoother\nuser experience will be valuable.\n\n[NodeSelector]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/\n[taint-and-toleration]: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n\n### Goals\n\n- Pods using a RuntimeClass that is not supported by all nodes in a cluster are\n  automatically scheduled to nodes that support that RuntimeClass.\n- RuntimeClass scheduling is compatible with other scheduling constraints. For\n  example, a pod with a node selector for GPUs and a Linux runtime should be\n  scheduled to a linux node with GPUs (an intersection).\n\n### Non-Goals\n\n- Replacing [SchedulingPolicy](#scheduling-policy) with RuntimeClasses.\n\nThe following are **currently** out of scope, but _may_ be revisited at a later\ndate.\n\n- Automatic topology discovery or node labeling\n- Automatically selecting a RuntimeClass for a pod based on node availability.\n- Defining official or reserved label or taint schemas or for RuntimeClasses.\n\n## Proposal\n\nA new optional `Scheduling` structure will be added to the RuntimeClass API. The\nscheduling struct includes both a `NodeSelector` and `Tolerations` that control\nhow a pod using that RuntimeClass is scheduled. The NodeSelector rules are\napplied during scheduling, but the Tolerations are added to the PodSpec during\nadmission by the new RuntimeClass admission controller.\n\n### User Stories\n\n#### Windows\n\nThe introduction of [Windows nodes][] presents an immediate use case for\nheterogeneous clusters, where some nodes are running Windows, and some\nlinux. From the inherent differences in the operating systems, it is natural\nthat each will support a different set of runtimes. For example, Windows nodes\nmay support Hyper-V sandboxing, while linux nodes support Kata-containers. Even\nnative container support varies on each, with runc for Linux and runhcs for\nWindows.\n\n- As a **cluster administrator** I want to enable different runtimes on Windows\n  and Linux nodes.\n- As a **developer** I want to select a Windows runtime without worrying about\n  scheduling constraints.\n- As a **developer** I want to ensure my Linux workloads are not accidentally\n  scheduled to windows nodes.\n\n[Windows nodes]: ../sig-windows/20190103-windows-node-support.md\n\n#### Sandboxed Nodes\n\nSome users wish to keep sandbox workloads and native workloads separate. For\nexample, a node running untrusted sandboxed workloads may have stricter\nrequirements about which trusted services are run on that node.\n\n- As a **cluster administrator** I want to ensure that untrusted workloads are\n  not colocated with sensitive data.\n- As a **developer** I want run an untrusted service without worrying about\n  where the service is running.\n- As a **cluster administrator** I want to autoscale trusted and untrusted nodes\n  independently.\n\n## Design Details\n\n### API\n\nThe RuntimeClass definition is augmented with an optional `Scheduling` struct:\n\n```go\ntype Scheduling struct {\n    // nodeSelector lists labels that must be present on nodes that support this\n    // RuntimeClass. Pods using this RuntimeClass can only be scheduled to a\n    // node matched by this selector. The RuntimeClass nodeSelector is merged\n    // with a pod's existing nodeSelector. Any conflicts will cause the pod to\n    // be rejected in admission.\n    // +optional\n    NodeSelector map[string]string\n\n    // tolerations adds tolerations to pods running with this RuntimeClass.\n    // +optional\n    Tolerations []corev1.Toleration\n}\n```\n\n**NodeSelector vs. NodeAffinity vs. NodeSelectorRequirement**\n\nThe PodSpec's `NodeSelector` is a label `map[string]string` that must exactly\nmatch a subset of node labels. [NodeAffinity][] is a much more complex and\nexpressive set of requirements and preferences. NodeSelectorRequirements are a\nsmall subset of the NodeAffinity rules, that place intersecting requirements on\na NodeSelectorTerm.\n\nSince the RuntimeClass scheduling rules represent hard requirements (the node\nsupports the RuntimeClass or it doesn't), the scheduling API should not include\npreferences, ruling out NodeAffinity. The NodeSelector type is much more\nexpressive than the `map[string]string` selector, but the top-level union logic\nmakes merging NodeSelectors messy (requires a cross-product). For simplicity,\nwe went with the simple requirements.\n\n[NodeAffinity]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n\n**Tolerations**\n\nWhile NodeSelectors and labels are used for steering pods _towards_ nodes,\n[taints and tolerations][taint-and-toleration] are used for steering pods _away_\nfrom nodes. If every pod had a RuntimeClass and every RuntimeClass had a strict\nNodeSelector, then RuntimeClasses could use non-overlapping selectors in place\nof taints \u0026 tolerations. However the same could be said of regular pod\nselectors, yet taints \u0026 tolerations are still a useful addition. Examples of\n[use cases](#user-stories) for including tolerations in RuntimeClass scheduling\ninculde:\n\n- Tainting Windows nodes with `windows.microsoft.com` to keep default linux pods\n  away from the nodes. Windows RuntimeClasses would then need a corresponding\n  toleration.\n- Tainting \"sandbox\" nodes with `sandboxed.kubernetes.io` to keep services\n  providing privileged node features away from sandboxed workloads. Sandboxed\n  RuntimeClasses would need a toleration to enable them to be run on those\n  nodes.\n\n### RuntimeClass Admission Controller\n\nThe RuntimeClass admission controller is a new default-enabled in-tree admission\nplugin. The role of the controller for scheduling is to merge the scheduling\nrules from the RuntimeClass into the PodSpec. Eventually, the controller's\nresponsibilities may grow, such as to merge in [pod overhead][] or validate\nfeature compatibility.\n\nMerging the RuntimeClass NodeSelector into the PodSpec NodeSelector is handled\nby adding the key-value pairs from the RuntimeClass version to the PodSpec\nversion. If both have the same key with a different value, then the admission\ncontroller will reject the pod.\n\nMerging tolerations is straight forward, as we want to _union_ the RuntimeClass\ntolerations with the existing tolerations, which matches the default toleration\ncomposition logic. This means that RuntimeClass tolerations can simply be\nappended to the existing tolerations, but an [existing\nutilty][merge-tolerations] can reduce duplicates by merging equivalent\ntolerations.\n\nIf the pod's referenced RuntimeClass does not exist, the admission controller\nwill reject the pod. This is necessary to ensure the pod is run with the\nexpected behavior.\n\n[merge-tolerations]: https://github.com/kubernetes/kubernetes/blob/58021216b16ae6d5f24fb1c32ab541b2e79a365e/pkg/util/tolerations/tolerations.go#L62\n[TaintBasedEvictions]: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#taint-based-evictions\n\n### Labeling Nodes\n\nNode labeling \u0026 tainting is outside the scope of this proposal or feature. How\nto label nodes is very environment dependent. Here are several examples:\n\n- If runtimes are setup as part of node setup, then the node template should\n  include the appropriate labels \u0026 taints.\n- If runtimes are installed through a DaemonSet, then the scheduling should match\n  that of the DaemonSet.\n- If runtimes are manually installed, or installed through some external\n  process, that same process should apply an appropriate label to the node.\n\nIf the RuntimeClass scheduling rules have security implications, special care\nshould be taken when choosing labels. In particular, labels with the\n`[*.]node-restriction.kubernetes.io/` prefix cannot be added with the node's\nidentity, and labels with the `[*.]k8s.io/` or `[*.]kubernetes.io/` prefixes\ncannot be modified by the node. For more details, see [Bounding Self-Labeling\nKubelets](../sig-auth/0000-20170814-bounding-self-labeling-kubelets.md)\n\n### Graduation Criteria\n\nThis feature will be rolled into RuntimeClass beta in v1.15, thereby skipping\nthe alpha phase. This means the feature is expected to be beta quality at launch:\n\n- Thorough testing, including unit, integration and e2e coverage.\n- Thoroughly documented (as an extension to the [RuntimeClass documentation][]).\n\n[RuntimeClass documentation]: https://kubernetes.io/docs/concepts/containers/runtime-class/\n\n## Implementation History\n\n- 2019-03-14: Initial KEP published.\n- 2018-10-05: [RuntimeClass Scheduling\n  Brainstorm](https://docs.google.com/document/d/1W51yBNTvp0taeEss56GTk8jczqFJ2d6jBeN6sCSlYZU/edit#)\n  published.\n\n## Alternatives\n\n### Scheduler\n\nA new scheduler predicate could manage the RuntimeClass scheduling. It would\nlookup the RuntimeClass associated with the pod being scheduled. If there is no\nRuntimeClass, or the RuntimeClass does not include a scheduling struct, then the\npredicate would permit the pod to be scheduled to the evaluated node. Otherwise,\nit would check whether the NodeSelector matches the node.\n\nAdding a dedicated RuntimeClass predicate rather than mixing the rules in to the\nNodeAffinity evaluation means that in the event a pod is unschedulable there\nwould be a clear explanation of why. For example:\n\n```\n0/10 Nodes are available: 5 nodes do not have enough memory, 5 nodes don't match RuntimeClass\n```\n\nIf the pod's referenced RuntimeClass does not exist at scheduling time, the\nRuntimeClass predicate would fail. The scheduler would periodically retry, and\nonce the RuntimeClass is (re)created, the pod would be scheduled.\n\n### RuntimeController Mix-in\n\nRather than resolving scheduling in the scheduler, the `NodeSelectorTerm`\nrules and `Tolerations` are mixed in to the PodSpec. The mix-in happens in the\nmutating admission phase, and is performed by a new `RuntimeController` built-in\nadmission plugin. The same admission controller is shared with the [Pod\nOverhead][] proposal.\n\n[Pod Overhead]: https://github.com/kubernetes/enhancements/pull/887\n\n#### RuntimeController\n\nRuntimeController is a new in-tree admission plugin that should eventually be\nenabled on almost every Kubernetes clusters. The role of the controller for\nscheduling is to merge the scheduling constraints from the RuntimeClass into the\nPodSpec. Eventually, the controller's responsibilities may grow, such as to\nmerge in [pod overhead][] or validate feature compatibility.\n\nNote that the RuntimeController is not needed if we implement [native scheduler\nsupport](#runtimeclass-aware-scheduling).\n\n#### Mix-in\n\nThe RuntimeClass scheduling rules are merged with the pod's NodeSelector \u0026\nTolerations.\n\n**NodeSelectorRequirements**\n\nTo avoid multiplicitive scaling of the NodeSelectorTerms, the\n`RuntimeClass.Scheduling.NodeSelector *v1.NodeSelector` field is replaced with\n`NodeSelectorTerm *v1.NodeSelectorTerm`.\n\nThe term's NodeSelectorRequirements are merged into the pod's node affinity\nscheduling requirements:\n\n```\npod.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[*].matchExpressions\n```\n\nSince the `requiredDuringSchedulingIgnoredDuringExecution` NodeSelectorTerms are\nunioned (OR'd), intersecting the RuntimeClass's NodeSelectorTerm means\nthe requirements need to be appended to _every_ NodeSelectorTerm.\n\n**Tolerations**\n\nMerging tolerations is much simpler as we want to _union_ the RuntimeClass\ntolerations with the existing tolerations, which matches the default toleration\ncomposition logic. This means that RuntimeClass tolerations can simply be\nappended to the existing tolerations, but an [existing\nutilty][merge-tolerations] can reduce duplicates by merging equivalent\ntolerations.\n\n[merge-tolerations]: https://github.com/kubernetes/kubernetes/blob/58021216b16ae6d5f24fb1c32ab541b2e79a365e/pkg/util/tolerations/tolerations.go#L62\n\n### NodeSelector\n\nReplacing the NodeSelector's `[]NodeSelectorRequirements` type with the\nPodSpec's label `map[string]string` approach greatly simplifies the merging\nlogic, but sacrifices a lot of flexibliity. For exameple, the operator in\nNodeSelectorRequriments enables selections like:\n\n- Negative selection, such as \"operating system is _not_ windows\"\n- Numerical comparison, such as \"runc version is _at least_ X\" (although it doesn't currently support semver)\n- Set selection, such as \"sandbox is _one of_ kata-cotainers or gvisor\"\n\n### Native RuntimeClass Reporting\n\nRather than relying on labels to stear RuntimeClasses to supporting nodes, nodes\ncould directly list supported RuntimeClasses (or RuntimeHandlers) in their\nstatus. Taking this approach would require native RuntimeClass-aware scheduling.\n\n**Advantages:**\n\n- RuntimeClass support is more explicit: it is easier to look at a node and see\n  which runtimes it supports.\n\n**Disadvantages:**\n\n- Larger change and more complexity: this requires modifying the node API and\n  introducing a new scheduling mechanism.\n- Less flexible: the existing scheduling mechanisms have been carefully thought\n  out and designed, and are extremely flexible to supporting a wide range of\n  topologies. Simple 1:1 matching would lose a lot of this flexibility.\n\nThe visibility advantage could be achieved through different methods. For\nexample, a special request or tool could be implemented that would list all the\nnodes that match a RuntimeClasses scheduling rules.\n\n### Scheduling Policy\n\nRather than building scheduling support into RuntimeClass, we could build\nRuntimeClass support into [SchedulingPolicy][]. For example, a scheduling\npolicy that places scheduling constraints on pods that use a particular\nRuntimeClass.\n\n**Advantages:**\n\n- A more generic system, no special logic needed for RuntimeClasses.\n- Scheduling constraints for correlated RuntimeClasses could be grouped together\n  (e.g. linux scheduling constraints for all linux RuntimeClasses).\n\n**Disadvantages:**\n\n- Separating the scheduling policy into a separate object means a less direct\n  user experience. The cluster administrator needs to setup 2 different\n  resources for each RuntimeClass, and the developer needs to look at 2\n  different resources to understand the full implications of choosing a\n  particular RuntimeClass.\n\nFor the same reason that RuntimeClass scheduling is compatible with additional\npod scheduling constraints, it should also be compatible with additional\nscheduling policies.\n\n[SchedulingPolicy]: https://github.com/kubernetes/enhancements/pull/683\n"
+  },
+  {
+    "id": "2f97c9eceec96ff1e7432969cf739a0b",
+    "title": "Runtime Class",
+    "authors": ["@tallclair"],
+    "owningSig": "sig-node",
+    "participatingSigs": ["sig-architecture"],
+    "reviewers": ["dchen1107", "derekwaynecarr", "yujuhong"],
+    "approvers": ["dchen1107", "derekwaynecarr"],
+    "editor": "",
+    "creationDate": "2018-06-19",
+    "lastUpdated": "2019-01-24",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Runtime Class\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n  - [User Stories](#user-stories)\n- [Proposal](#proposal)\n  - [API](#api)\n    - [Examples](#examples)\n    - [Runtime Handler](#runtime-handler)\n  - [Versioning, Updates, and Rollouts](#versioning-updates-and-rollouts)\n  - [Implementation Details](#implementation-details)\n    - [Monitoring](#monitoring)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Appendix](#appendix)\n  - [Proposed Future Enhancements](#proposed-future-enhancements)\n  - [Examples of runtime variation](#examples-of-runtime-variation)\n\u003c!-- /toc --\u003e\n\n## Summary\n\n`RuntimeClass` is a new cluster-scoped resource that surfaces container runtime properties to the\ncontrol plane. RuntimeClasses are assigned to pods through a `runtimeClass` field on the\n`PodSpec`. This provides a new mechanism for supporting multiple runtimes in a cluster and/or node.\n\n## Motivation\n\nThere is growing interest in using different runtimes within a cluster. [Sandboxes][] are the\nprimary motivator for this right now, with both Kata containers and gVisor looking to integrate with\nKubernetes. Other runtime models such as Windows containers or even remote runtimes will also\nrequire support in the future. RuntimeClass provides a way to select between different runtimes\nconfigured in the cluster and surface their properties (both to the cluster \u0026 the user).\n\nIn addition to selecting the runtime to use, supporting multiple runtimes raises other problems to\nthe control plane level, including: accounting for runtime overhead, scheduling to nodes that\nsupport the runtime, and surfacing which optional features are supported by different\nruntimes. Although these problems are not tackled by this initial proposal, RuntimeClass provides a\ncluster-scoped resource tied to the runtime that can help solve these problems in a future update.\n\n[Sandboxes]: https://docs.google.com/document/d/1QQ5u1RBDLXWvC8K3pscTtTRThsOeBSts_imYEoRyw8A/edit\n\n### Goals\n\n- Provide a mechanism for surfacing container runtime properties to the control plane\n- Support multiple runtimes per-cluster, and provide a mechanism for users to select the desired\n  runtime\n\n### Non-Goals\n\n- RuntimeClass is NOT RuntimeComponentConfig.\n- RuntimeClass is NOT a general policy mechanism.\n- RuntimeClass is NOT \"NodeClass\". Although different nodes may run different runtimes, in general\n  RuntimeClass should not be a cross product of runtime properties and node properties.\n\n### User Stories\n\n- As a cluster operator, I want to provide multiple runtime options to support a wide variety of\n  workloads. Examples include native linux containers, \"sandboxed\" containers, and windows\n  containers.\n- As a cluster operator, I want to provide stable rolling upgrades of runtimes. For\n  example, rolling out an update with backwards incompatible changes or previously unsupported\n  features.\n- As an application developer, I want to select the runtime that best fits my workload.\n- As an application developer, I don't want to study the nitty-gritty details of different runtime\n  implementations, but rather choose from pre-configured classes.\n- As an application developer, I want my application to be portable across clusters that use similar\n  but different variants of a \"class\" of runtimes.\n\n## Proposal\n\nThe initial design includes:\n\n- `RuntimeClass` API resource definition\n- `RuntimeClass` pod field for specifying the RuntimeClass the pod should be run with\n- Kubelet implementation for fetching \u0026 interpreting the RuntimeClass\n- CRI API \u0026 implementation for passing along the [RuntimeHandler](#runtime-handler).\n\n### API\n\n`RuntimeClass` is a new cluster-scoped resource in the `node.k8s.io` API group.\n\n\u003e _The `node.k8s.io` API group would eventually hold the Node resource when `core` is retired.\n\u003e Alternatives considered: `runtime.k8s.io`, `cluster.k8s.io`_\n\n_(This is a simplified declaration, syntactic details will be covered in the API PR review)_\n\n```go\ntype RuntimeClass struct {\n    metav1.TypeMeta\n    // ObjectMeta minimally includes the RuntimeClass name, which is used to reference the class.\n    // Namespace should be left blank.\n    metav1.ObjectMeta\n\n    Spec RuntimeClassSpec\n}\n\ntype RuntimeClassSpec struct {\n    // RuntimeHandler specifies the underlying runtime the CRI calls to handle pod and/or container\n    // creation. The possible values are specific to a given configuration \u0026 CRI implementation.\n    // The empty string is equivalent to the default behavior.\n    // +optional\n    RuntimeHandler string\n}\n```\n\nThe runtime is selected by the pod by specifying the RuntimeClass in the PodSpec. Once the pod is\nscheduled, the RuntimeClass cannot be changed.\n\n```go\ntype PodSpec struct {\n    ...\n    // RuntimeClassName refers to a RuntimeClass object with the same name,\n    // which should be used to run this pod.\n    // +optional\n    RuntimeClassName string\n    ...\n}\n```\n\nAn unspecified `nil` or empty `\"\"` RuntimeClassName is equivalent to the backwards-compatible\ndefault behavior as if the RuntimeClass feature is disabled.\n\n#### Examples\n\nSuppose we operate a cluster that lets users choose between native runc containers, and gvisor and\nkata-container sandboxes. We might create the following runtime classes:\n\n```yaml\nkind: RuntimeClass\napiVersion: node.k8s.io/v1alpha1\nmetadata:\n    name: native  # equivalent to 'legacy' for now\nspec:\n    runtimeHandler: runc\nkind: RuntimeClass\napiVersion: node.k8s.io/v1alpha1\nmetadata:\n    name: gvisor\nspec:\n    runtimeHandler: gvisor\nkind: RuntimeClass\napiVersion: node.k8s.io/v1alpha1\nmetadata:\n    name: kata-containers\nspec:\n    runtimeHandler: kata-containers\n# provides the default sandbox runtime when users don't care about which they're getting.\nkind: RuntimeClass\napiVersion: node.k8s.io/v1alpha1\nmetadata:\n  name: sandboxed\nspec:\n  runtimeHandler: gvisor\n```\n\nThen when a user creates a workload, they can choose the desired runtime class to use (or not, if\nthey want the default).\n\n```yaml\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n  name: sandboxed-nginx\nspec:\n  replicas: 2\n  selector:\n    matchLabels:\n      app: sandboxed-nginx\n  template:\n    metadata:\n      labels:\n        app: sandboxed-nginx\n    spec:\n      containers:\n      - name: nginx\n        image: nginx\n        ports:\n        - containerPort: 80\n          protocol: TCP\n```\n\n#### Runtime Handler\n\nThe `RuntimeHandler` is passed to the CRI as part of the `RunPodSandboxRequest`:\n\n```proto\nmessage RunPodSandboxRequest {\n    // Configuration for creating a PodSandbox.\n    PodSandboxConfig config = 1;\n    // Named runtime configuration to use for this PodSandbox.\n    string RuntimeHandler = 2;\n}\n```\n\nThe RuntimeHandler is provided as a mechanism for CRI implementations to select between different\npredetermined configurations. The initial use case is replacing the experimental pod annotations\ncurrently used for selecting a sandboxed runtime by various CRI implementations:\n\n| CRI Runtime | Pod Annotation                                              |\n| CRIO        | io.kubernetes.cri-o.TrustedSandbox: \"false\"                 |\n| containerd  | io.kubernetes.cri.untrusted-workload: \"true\"                |\n| frakti      | runtime.frakti.alpha.kubernetes.io/OSContainer: \"true\"\u003cbr\u003eruntime.frakti.alpha.kubernetes.io/Unikernel: \"true\" |\n| windows     | experimental.windows.kubernetes.io/isolation-type: \"hyperv\" |\n\nThese implementations could stick with scheme (\"trusted\" and \"untrusted\"), but the preferred\napproach is a non-binary one wherein arbitrary handlers can be configured with a name that can be\nmatched against the specified RuntimeHandler. For example, containerd might have a configuration\ncorresponding to a \"kata-runtime\" handler:\n\n```\n[plugins.cri.containerd.kata-runtime]\n    runtime_type = \"io.containerd.runtime.v1.linux\"\n    runtime_engine = \"/opt/kata/bin/kata-runtime\"\n    runtime_root = \"\"\n```\n\nThis non-binary approach is more flexible: it can still map to a binary RuntimeClass selection\n(e.g. `sandboxed` or `untrusted` RuntimeClasses), but can also support multiple parallel sandbox\ntypes (e.g. `kata-containers` or `gvisor` RuntimeClasses).\n\n### Versioning, Updates, and Rollouts\n\nRuntimes are expected to be managed by the cluster administrator (or provisioner). In most cases,\nruntime upgrades (and downgrades) should be handled by the administrator, without requiring any\ninteraction from the user. In these cases, the runtimes can be treated the same way we treat other\nnode components such as the Kubelet, node OS, and CRI runtime. In other words, the upgrade process\nmeans rolling out nodes with the updated runtime, and gradually draining and removing old nodes from\nthe pool. For more details, see [Maintenance on a\nNode](https://kubernetes.io/docs/tasks/administer-cluster/cluster-management/#maintenance-on-a-node).\n\nIf the upgraded runtime includes new features that users wish to take advantage of immediately, then\nnode labels can be used to select nodes supporting the updated runtime. In the uncommon scenario\nwhere substantial changes to the runtime are made and application changes may be required, we\nrecommend that the updated runtime be treated as a _new_ runtime, with a separate RuntimeClass\n(e.g. `sandboxed-v2`). This approach has the advantage of native support for rolling updates through\nthe same mechanisms as any other application update, so the updated applications can be carefully\nrolled out to the new runtime.\n\nRuntime upgrades will benefit from better scheduling support, which is a feature we plan to add in a\nfuture release.\n\n### Implementation Details\n\nThe Kubelet uses an Informer to keep a local cache of all RuntimeClass objects. When a new pod is\nadded, the Kubelet resolves the Pod's RuntimeClass against the local RuntimeClass cache.  Once\nresolved, the RuntimeHandler field is passed to the CRI as part of the\n[`RunPodSandboxRequest`][runpodsandbox]. At that point, the interpretation of the RuntimeHandler is\nleft to the CRI implementation, but it should be cached if needed for subsequent calls.\n\nIf the RuntimeClass cannot be resolved (e.g. doesn't exist) at Pod creation, then the request will\nbe rejected in admission (controller to be detailed in a following update). If the RuntimeClass\ncannot be resolved by the Kubelet when `RunPodSandbox` should be called, then the Kubelet will fail\nthe Pod. The admission check on a replica recreation will prevent the scheduler from thrashing. If\nthe `RuntimeHandler` is not recognized by the CRI implementation, then `RunPodSandbox` will return\nan error.\n\n[runpodsandbox]: https://github.com/kubernetes/kubernetes/blob/b05a61e299777c2030fbcf27a396aff21b35f01b/pkg/kubelet/apis/cri/runtime/v1alpha2/api.proto#L344\n\n#### Monitoring\n\nThe first round of monitoring implementation for `RuntimeClass` covers the\nfollowing two areas and is finished (tracked in\n[#73058](https://github.com/kubernetes/kubernetes/issues/73058)):\n\n- `how robust is every runtime?` A new metric\n  [RunPodSandboxErrors](https://github.com/kubernetes/kubernetes/blob/596a48dd64bcaa01c1d2515dc79a558a4466d463/pkg/kubelet/metrics/metrics.go#L351)\n  is added to track the RunPodSandbox operation errors, broken down by\n  RuntimeClass.\n- `how expensive is every runtime in terms of latency?` A new metric\n  [RunPodSandboxDuration](https://github.com/kubernetes/kubernetes/blob/596a48dd64bcaa01c1d2515dc79a558a4466d463/pkg/kubelet/metrics/metrics.go#L341)\n  is added to track the duration of RunPodSandbox operations, broken down by\n  RuntimeClass.\n\nThe following monitoring areas will be skipped for now, but may be considered\nafter the RuntimeClass scheduling is implemented:\n\n- how many runtimes does a cluster support?\n- how many scheduling failures were caused by unsupported runtimes or insufficient\n  resources of a certain runtime?\n\nCurrently, we assume that all the nodes in a cluster are homogeneous. After\nheterogeneous clusters are implemented, we may need to monitor how many runtimes\na node supports.\n\n### Risks and Mitigations\n\n**Scope creep.** RuntimeClass has a fairly broad charter, but it should not become a default\ndumping ground for every new feature exposed by the node. For each feature, careful consideration\nshould be made about whether it belongs on the Pod, Node, RuntimeClass, or some other resource. The\n[non-goals](#non-goals) should be kept in mind when considering RuntimeClass features.\n\n**Becoming a general policy mechanism.** RuntimeClass should not be used a replacement for\nPodSecurityPolicy. The use cases for defining multiple RuntimeClasses for the same underlying\nruntime implementation should be extremely limited (generally only around updates \u0026 rollouts). To\nenforce this, no authorization or restrictions are placed directly on RuntimeClass use; in order to\nrestrict a user to a specific RuntimeClass, you must use another policy mechanism such as\nPodSecurityPolicy.\n\n**Pushing complexity to the user.** RuntimeClass is a new resource in order to hide the complexity\nof runtime configuration from most users (aside from the cluster admin or provisioner). However, we\nare still side-stepping the issue of precisely defining specific types of runtimes like\n\"Sandboxed\". However, it is still up for debate whether precisely defining such runtime categories\nis even possible. RuntimeClass allows us to decouple this specification from the implementation, but\nit is still something I hope we can address in a future iteration through the concept of pre-defined\nor \"conformant\" RuntimeClasses.\n\n**Non-portability.** We are already in a world of non-portability for many features (see [examples\nof runtime variation](#examples-of-runtime-variation). Future improvements to RuntimeClass can help\naddress this issue by formally declaring supported features, or matching the runtime that supports a\ngiven workload automitaclly. Another issue is that pods need to refer to a RuntimeClass by name,\nwhich may not be defined in every cluster. This is something that can be addressed through\npre-defined runtime classes (see previous risk), and/or by \"fitting\" pod requirements to compatible\nRuntimeClasses.\n\n## Graduation Criteria\n\nAlpha:\n\n- [x] Everything described in the current proposal:\n  - [x] Introduce the RuntimeClass API resource\n  - [x] Add a RuntimeClassName field to the PodSpec\n  - [x] Add a RuntimeHandler field to the CRI `RunPodSandboxRequest`\n  - [x] Lookup the RuntimeClass for pods \u0026 plumb through the RuntimeHandler in the Kubelet (feature\n    gated)\n- [x] RuntimeClass support in at least one CRI runtime \u0026 dockershim\n  - [x] Runtime Handlers can be statically configured by the runtime, and referenced via RuntimeClass\n  - [x] An error is reported when the handler or is unknown or unsupported\n- [x] Testing\n  - [x] Kubernetes E2E tests (only validating single runtime handler cases)\n\nBeta:\n\n- [x] Several major runtimes support RuntimeClass, and the current [untrusted annotations](#runtime-handler) are\n  deprecated.\n  - [x] [containerd](https://github.com/containerd/cri/pull/891)\n  - [x] [CRI-O](https://github.com/kubernetes-sigs/cri-o/pull/1847)\n  - [x] [dockershim](https://github.com/kubernetes/kubernetes/pull/67909)\n- [ ] Comprehensive test coverage\n  - [ ] [CRI validation tests][cri-validation]\n  - [ ] RuntimeClasses are configured in the E2E environment with test coverage of a non-default\n    RuntimeClass\n- [x] Comprehensive coverage of RuntimeClass metrics. [#73058](http://issue.k8s.io/73058)\n- [x] The update \u0026 upgrade story is revisited, and a longer-term approach is implemented as necessary.\n\n[cri-validation]: https://github.com/kubernetes-sigs/cri-tools/blob/master/docs/validation.md\n\n## Implementation History\n\n- 2018-09-27: RuntimeClass released as alpha with Kubernetes v1.12\n- 2018-06-11: SIG-Node decision to move forward with proposal\n- 2018-06-19: Initial KEP published.\n\n## Appendix\n\n### Proposed Future Enhancements\n\nThe following ideas may be explored in a future iteration:\n\n- Surfacing support for optional features by runtimes, and surfacing errors caused by\n  incompatible features \u0026 runtimes earlier.\n- Automatic runtime or feature discovery - initially RuntimeClasses are manually defined (by the\n  cluster admin or provider), and are asserted to be an accurate representation of the runtime.\n- Scheduling in heterogeneous clusters - it is possible to operate a heterogeneous cluster\n  (different runtime configurations on different nodes) through scheduling primitives like\n  `NodeAffinity` and `Taints+Tolerations`, but the user is responsible for setting these up and\n  automatic runtime-aware scheduling is out-of-scope.\n- Define standardized or conformant runtime classes - although I would like to declare some\n  predefined RuntimeClasses with specific properties, doing so is out-of-scope for this initial KEP.\n- [Pod Overhead][] - Although RuntimeClass is likely to be the configuration mechanism of choice,\n  the details of how pod resource overhead will be implemented is out of scope for this KEP.\n- Provide a mechanism to dynamically register or provision additional runtimes.\n- Requiring specific RuntimeClasses according to policy. This should be addressed by other\n  cluster-level policy mechanisms, such as PodSecurityPolicy.\n- \"Fitting\" a RuntimeClass to pod requirements - In other words, specifying runtime properties and\n  letting the system match an appropriate RuntimeClass, rather than explicitly assigning a\n  RuntimeClass by name. This approach can increase portability, but can be added seamlessly in a\n  future iteration.\n- The cluster admin can choose which RuntimeClass is the default in a cluster.\n\n[Pod Overhead]: https://docs.google.com/document/d/1EJKT4gyl58-kzt2bnwkv08MIUZ6lkDpXcxkHqCvvAp4/edit\n\n### Examples of runtime variation\n\n- Linux Security Module (LSM) choice - Kubernetes supports both AppArmor \u0026 SELinux options on pods,\n  but those are mutually exclusive, and support of either is not required by the runtime. The\n  default configuration is also not well defined.\n- Seccomp-bpf - Kubernetes has alpha support for specifying a seccomp profile, but the default is\n  defined by the runtime, and support is not guaranteed.\n- Windows containers - isolation features are very OS-specific, and most of the current features are\n  limited to linux. As we build out Windows container support, we'll need to add windows-specific\n  features as well.\n- Host namespaces (Network,PID,IPC) may not be supported by virtualization-based runtimes\n  (e.g. Kata-containers \u0026 gVisor).\n- Per-pod and Per-container resource overhead varies by runtime.\n- Device support (e.g. GPUs) varies wildly by runtime \u0026 nodes.\n- Supported volume types varies by node - it remains TBD whether this information belongs in\n  RuntimeClass.\n- The list of default capabilities is defined in Docker, but not Kubernetes. Future runtimes may\n  have differing defaults, or support a subset of capabilities.\n- `Privileged` mode is not well defined, and thus may have differing implementations.\n- Support for resource over-commit and dynamic resource sizing (e.g. Burstable vs Guaranteed\n  workloads)\n"
+  },
+  {
+    "id": "0d0329288a59e7b722eb5f5b95e74934",
+    "title": "Kubernetes Community Artifact Serving",
+    "authors": ["@brendandburns"],
+    "owningSig": "sig-release",
+    "participatingSigs": ["sig-architecture", "sig-contributor-experience", "sig-testing"],
+    "reviewers": ["@justinsb", "@dims"],
+    "approvers": ["@dims"],
+    "editor": "@brendandburns",
+    "creationDate": "2019-01-23",
+    "lastUpdated": "2019-01-31",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Kubernetes Artifact Management\n\n1. **Fill out the \"overview\" sections.**\n  This includes the Summary and Motivation sections.\n  These should be easy if you've preflighted the idea of the KEP with the appropriate SIG.\n1. **Create a PR.**\n  Assign it to folks in the SIG that are sponsoring this process.\n1. **Merge early.**\n  Avoid getting hung up on specific details and instead aim to get the goal of the KEP merged quickly.\n  The best way to do this is to just start with the \"Overview\" sections and fill out details incrementally in follow on PRs.\n  View anything marked as a `provisional` as a working document and subject to change.\n  Aim for single topic PRs to keep discussions focused.\n  If you disagree with what is already in a document, open a new PR with suggested changes.\n\nThe canonical place for the latest set of instructions (and the likely source of this file) is [here](/keps/YYYYMMDD-kep-template.md).\n\nThe `Metadata` section above is intended to support the creation of tooling around the KEP process.\nThis will be a YAML section that is fenced as a code block.\nSee the KEP process for details on each of these items.\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [HTTP Redirector Design](#http-redirector-design)\n    - [Configuring the HTTP Redirector](#configuring-the-http-redirector)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\nThis document describes how official artifacts (Container Images, Binaries) for the Kubernetes\nproject are managed and distributed.\n\n\n## Motivation\n\nThe motivation for this KEP is to describe a process by which artifacts (container images, binaries)\ncan be distributed by the community. Currently the process by which images is both ad-hoc in nature\nand limited to an arbitrary set of people who have the keys to the relevant repositories. Standardize\naccess will ensure that people around the world have access to the same artifacts by the same names\nand that anyone in the project is capable (if given the right authority) to distribute images.\n\n### Goals\n\nThe goals of this process are to enable:\n  * Anyone in the community (with the right permissions) to manage the distribution of Kubernetes images and binaries.\n  * Fast, cost-efficient access to artifacts around the world through appropriate mirrors and distribution\n\nThis KEP will have succeeded when artifacts are all managed in the same manner and anyone in the community\n(with the right permissions) can manage these artifacts.\n\n### Non-Goals\n\nThe actual process and tooling for promoting images, building packages or otherwise assembling artifacts\nis beyond the scope of this KEP. This KEP deals with the infrastructure for serving these things via\nHTTP as well as a generic description of how promotion will be accomplished.\n\n## Proposal\n\nThe top level design will be to set up a global redirector HTTP service (`artifacts.k8s.io`) \nwhich knows how to serve HTTP and redirect requests to an appropriate mirror. This redirector\nwill serve both binary and container image downloads. For container images, the HTTP redirector\nwill redirect users to the appropriate geo-located container registry. For binary artifacts, \nthe HTTP redirector will redirect to appropriate geo-located storage buckets.\n\nTo facilitate artifact promotion, each project, as necessary, will be given access to a\nproject staging area relevant to their particular artifacts (either storage bucket or image \nregistry). Each project is free to manage their assets in the staging area however they feel\nit is best to do so. However, end-users are not expected to access artifacts through the\nstaging area.\n\nFor each artifact, there will be a configuration file checked into this repository. When a\nproject wants to promote an image, they will file a PR in this repository to update their\nimage promotion configuration to promote an artifact from staging to production. Once this\nPR is approved, automation that is running in the k8s project infrastructure (e.g. \nhttps://github.com/GoogleCloudPlatform/k8s-container-image-promoter) will pick up this new\nconfiguration file and copy the relevant bits out to the production serving locations.\n\nImportantly, if a project needs to roll-back or remove an artifact, the same process will\noccur, so that the promotion tool needs to be capable of deleting images and artifacts as\nwell as promoting them.\n\n### HTTP Redirector Design\nTo facilitate world-wide distribution of artifacts from a single (virtual) location we will\nideally run a replicated redirector service in the United States, Europe and Asia.\nEach of these redirectors\nservices will be deployed in a Kubernetes cluster and they will be exposed via a public IP\naddress and a dns record indicating their location (e.g. `europe.artifacts.k8s.io`).\n\nWe will use Geo DNS to route requests to `artifacts.k8s.io` to the correct redirector. This is necessary to ensure that we always route to a server which is accessible no matter what region we are in. We will need to extend or enhance the existing DNS synchronization tooling to handle creation of the GeoDNS records.\n\n#### Configuring the HTTP Redirector\nTHe HTTP Redirector service will be driven from a YAML configuration that specifies a path to mirror\nmapping. For now the redirector will serve content based on continent, for example:\n\n```yaml\n/kops\n  - Americas: americas.artifacts.k8s.io\n  - Asia: asia.artifacts.k8s.io\n  - default: americas.artificats.k8s.io\n```\n\nThe redirector will use this data to redirect a request to the relevant mirror using HTTP 302 responses. The implementation of the mirrors themselves are details left to the service implementor and may be different depending on the artifacts being exposed (binaries vs. container images)\n\n## Graduation Criteria\n\nThis KEP will graduate when the process is implemented and has been successfully used to\nmanage the images for a Kubernetes release.\n\n## Implementation History\n\nNone yet.\n\n"
+  },
+  {
+    "id": "5fdd23a89a36fdf470a00462dde80f25",
+    "title": "Publishing kubernetes packages",
+    "authors": ["@hoegaarden"],
+    "owningSig": "sig-release",
+    "participatingSigs": ["sig-cluster-lifecycle"],
+    "reviewers": ["@timothysc", "@sumitranr", "@Klaven", "@ncdc", "@ixdy", "“@neolit123”"],
+    "approvers": ["@spiffxp", "@tpepper"],
+    "editor": "TBD",
+    "creationDate": "2019-02-19",
+    "lastUpdated": "2019-02-27",
+    "status": "provisional",
+    "seeAlso": [
+      "https://github.com/kubernetes/enhancements/pull/858",
+      "/keps/sig-release/20190121-artifact-management.md",
+      "/keps/sig-release/k8s-image-promoter.md"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Publishing kubernetes packages\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n    - [Alpha](#alpha)\n    - [Removing deprecated publishing artifacts](#removing-deprecated-publishing-artifacts)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Drawbacks [optional]](#drawbacks-optional)\n- [Alternatives [optional]](#alternatives-optional)\n- [Infrastructure Needed](#infrastructure-needed)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n\n## Summary\n\nThis document describes how deb \u0026 rpm packages get published as part of cutting a release, with the tooling the release and patch management teams have at hand ([anago], [gcbmgr] and others from [k/release], [k/k/build] and potentially other places).\n\n[anago]: https://github.com/kubernetes/release/tree/master/anago\n[gcbmgr]: https://github.com/kubernetes/release/tree/master/gdcbmgr\n[k/release]: https://github.com/kubernetes/release\n[k/k/build]: https://github.com/kubernetes/kubernetes/tree/master/build\n\n## Motivation\n\nCurrently ...\n- Only Googlers can publish packages.\n- Package spec file updates are not committed to a public repository.\n- The packages get published on Google infrastructure.\n- After publishing a new release but before sending out the release\n  notification emails the process needs to be paused. Googlers need to build\n  and publish the deb and rpm packages before the branch management team can\n  continue and send notification can be sent out.\n- We can only publish packages for stable releases right now.\n- We use different packages in CI then we officially release.\n\nThis all prolongs the release process, it is a hard dependency on a small group of people from one company (and its infrastructure), and we only ever publish and test packages very late in the release process.\n\n### Goals\n\nThe whole process should be folded into the release tooling, it should be part of the release process, and should not involve anyone other than the release branch / patch release team.\nFor each release the release team cuts, packages should also be generated and published automatically.\n\nThere should be multiple channels users can subscribe to: **stable**, **dev**, and **nightly**.\n\n\n### Non-Goals\n\nThe actual package generation is a different problem that is discussed in [this KEP][pkg-gen-kep].\n\n[pkg-gen-kep]: https://github.com/kubernetes/enhancements/pull/858\n\n## Proposal\n\n- Make the infrastructure generic and simple enough to be easily handed off to the CNCF\n    - Storage buckets (to store the staged/released packages)\n    - DNS entries (e.g. apt.kubernetes.io, ...)\n    - package mirror (e.g. a self hosted aptly/artifactory/... or as a service)\n        - have multiple channels, e.g. `stable`, `dev`, `nightly`\n- Run the [package building][pkg-gen-kep] as part of the staging process\n    - on GCB / not on an individual's machine\n- Have a safe way to store the signing key and make it available to the release team and release tooling\n- Automatically sign the repository and packages\n- Automatically build and publish packages on a nightly basis\n\n\n### User Stories\n\n*Note*: The following user stories are using the keywords of the [gherkin language][gherkin].\n\n[gherkin]: https://docs.cucumber.io/gherkin/reference/\n\n\n```\nScenario: Enduser installs a kubelet from the stable channel\n  Given a user has configured the officially documented package mirror for stable releases for a specific kubernetes minor version (\"the minor\") on their machine\n   When they use the system's package manager to query the list of kubelet packages available (e.g. apt-cache policy kubelet)\n   Then they see a list of all stable patch versions of the kubelet that stem from the minor and a preference to install the latest patch version of the kubelet\n    But don't see any alpha, beta, rc or nightly releases of the kubelet from this specific kubernetes minor version\n    And they don't see any packages of any other kubernetes minor release\n```\n\n```\nScenario: Release tools automatically publish new packages\n  Given a release team member ran `./gcbmgr stage master --build-at-head --nomock`\n    And a release team member ran `./gcbmgr release master --buildversion=${{VERSIONID}} --nomock`\n   When a user inspects the officially documented deb and rpm kubernetes repositories for this specific kubernetes minor version\n   Then they see the newly cut alpha releases published in the alpha channel only\n```\n\n```\nScenario: End users can get the public key the packages or the repository metadata is signed with\n  Given a user has a system configured with not allowing unsigned untrusted package repositories\n    And they have setup the officially documented repository for a specific kubernetes minor release\n   When they download the public key from the location stated in the official documentation\n    And they configure their system's package manager to use that key\n    And they use their system's package manager to install a package from this specific kubernetes minor release\n   Then their package manager will not complain about untrusted packages, sources or repositories\n```\n\n\u003c!--\n```\nScenario: [...]\n  Given ...\n    And ...\n   When ...\n   Then ...\n```\n--\u003e\n\n### Implementation Details/Notes/Constraints\n\nPackages will be published for different ...\n- **`${dist}`**: a combination of `\u003cdistribution\u003e-\u003ccode-name-or-version\u003e`\n  (e.g. `debian-jessie`, `ubuntu-xenial`, `fedora-23`, `centos-7`, ...)\n- **`${k8s_release}`**: the version of kubernetes `\u003cmajor\u003e.\u003cminor\u003e`\n  (e.g. `1.12`, `1.13`, `1.14`, ...)\n- **`${channel}`**: can be `stable`, `dev`, `nightly`\n    - `stable`: all official releases for `${k8s_release}`\n      (e.g.: `1.13.0`, `1.13.1`, `1.13.2`, ...)\n    - `dev`: all development releases for all minor releases in this `${k8s_release}`, including `alpha`s, `beta`s and `rc`s\n      (e.g.: `1.13.0-rc.2`, `1.13.2-beta.0`, `1.13.1-alpha.3`, ...)\n    - `nightly`: any package cut automatically on a daily basis\n\n\nThis means, that end-users can configure their systems’ package managers to use those different `${channel}`s of a kubernetes `${k8s_release}` for their `${dist}`.\n\nA configuration for the package managers might look something like:\n\n- deb:\n    ```\n    # deb http://apt.kubernetes.io/${dist} ${k8s_release} ${channel}\n    deb http://apt.kubernetes.io/debian-jessie 1.13 nightly\n    ```\n- rpm/yum:\n    ```\n    [kubernetes]\n    name=Kubernetes\n    # baseurl=http://yum.kubernetes.io/${dist}/${k8s_release}/${channel}\n    baseurl=http://yum.kubernetes.io/fedora-27/1.13/nightly\n    enabled=1\n    gpgcheck=1\n    repo_gpgcheck=1\n    gpgkey=file:///etc/pki/rpm-gpg/kubernetes.gpg.pub\n    ```\n\nDifferent architectures will be published into the same repos, it is up to the package managers to pull and install the correct package for the target platform.\n\n\nIdeally, publishing/promoting a package means to commit a change to a configuration file which triggers a \"package promotion tool\".\nThat tool ...\n- manages which packages need to go into which `${channel}` for which `${dist}` of which `${k8s_release}`\n- guard that by the packages checksum\n- is able to promote a package from a bucket and also from a `${channel}` to the other\n- work off of a declarative configuration\n\nThis tool does for packages what the [Image Promoter][img-promoter] tool does\nfor container images. Therefore, ideally, we can implement this work-flow as part\nof the [Image Promoter][img-promoter] or at least use its libraries.\n\n\nAs soon as the [redirector] is in place the repositories should be mirrored and the [redirector] should be used.\n\n[redirector]: https://github.com/kubernetes/enhancements/blob/master/keps/sig-release/20190121-artifact-management.md#http-redirector-design\n[img-promoter]: https://github.com/kubernetes/enhancements/blob/7a2e7c25ee3f2a50f2218557801fbd8dd79fd0f2/keps/sig-release/k8s-image-promoter.md\n\nAll architectures that are supported by the [package building tool][pkg-gen-kep] should be published.\nThis KEP suggests to start with publishing a single supported architecture\n(e.g. `linux/amd64`) and extend that iteratively, when we verify that creating\nall packages for all architectures is fast enough to be done as part of the\nrelease process. If it turns out this step takes too long, we need to think\nabout doing the package building \u0026 publishing asynchronous to the release\nprocess (see also: [Risks](#risks-and-mitigations)).\n\n### Risks and Mitigations\n\n- *Risk*: We don't find a proper way to share secrets like the signing key  \n  *Mitigation*: TBA\n- *Risk*: Building all the packages for all the distributions and their version takes too long to be done nightly or via cutting the release  \n  *Mitigation*: TBA\n\n## Design Details\n\n### Test Plan\n\nThere should be a post-publish tests, which can be run as part or after the release process\n- pull packages from the  official mirrors (via the [redirector] if in place)\n- assert that all the packages we expect to be published are actually published\n- assert that the packages and the repo metadata is signed with the current signing key\n\n### Graduation Criteria\n\nIn general we can keep the current way of publishing (via googlers onto google's infrastructure) and introduce new infrastructure in parallel.\n\nOnce the tests show that the mirrors are good, we can adapt the official documentation. This includes:\n- for release team members:\n  - How and where do the packages get published as part of the release process\n  - How can the post-publish test be run\n- for kubernetes contributors:\n  - How and where do the nightlies get published\n- for kubernetes users:\n  - Which repository are available for users\n  - How to configure their package managers\n\n\n\n\n#### Alpha\n\n- Needed infrastructure is in place (buckets, dns, repos, …)\n\n#### Alpha -\u003e Beta Graduation\n\n\n- [anago] [creates packages][pkg-gen-kep] and published those packages as part of the release process\n- post-publish tests are in places and run as part of the release process\n- nightlies will be build and published on a daily basis\n- documentation is in place\n\n#### Beta -\u003e GA Graduation\n\n\nThis new publishing infrastructure and mechanisms can be considered GA when no googler is needed anymore to publish the packages.\n\n#### Removing deprecated publishing artifacts\n\nWhen 2 releases have been cut and published with the new mechanism any of the older tools and processes (e.g. `k/release/{deb,rpm}`) can be removed.\n\nN/A\n\n### Upgrade / Downgrade Strategy\n\nN/A\n\n### Version Skew Strategy\n\nN/A\n\n\n## Implementation History\n\n\u003c!--\n- the `Summary` and `Motivation` sections being merged signaling SIG acceptance\n- the `Proposal` section being merged signaling agreement on a proposed design\n- the date implementation started\n- the first Kubernetes release where an initial version of the KEP was available\n- the version of Kubernetes where the KEP graduated to general availability\n- when the KEP was retired or superseded\n--\u003e\n\nTBA\n\n## Drawbacks [optional]\n\nN/A\n\n## Alternatives [optional]\n\nN/A\n\n## Infrastructure Needed\n\nTBA\n"
+  },
+  {
+    "id": "1fde4a2e1f67b0133c57d5a9807f0d12",
+    "title": "Rebase Kubernetes Main Master and Node Images to Distroless/static",
+    "authors": ["@yuwenma"],
+    "owningSig": "sig-release",
+    "participatingSigs": ["sig-release", "sig-cloud-provider"],
+    "reviewers": ["@tallclair"],
+    "approvers": ["@tallclair"],
+    "editor": "yuwenma",
+    "creationDate": "2019-03-16",
+    "lastUpdated": "2019-03-21",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Rebase Kubernetes Main Master and Node Images to Distroless/static\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Background](#background)\n  - [Kubernetes Images](#kubernetes-images)\n    - [Type 1 FROM Scratch](#type-1-from-scratch)\n    - [Type 2 Debian Based Images](#type-2-debian-based-images)\n    - [Type 3 Alpine Based Images](#type-3-alpine-based-images)\n  - [Distroless and Previous Work](#distroless-and-previous-work)\n- [Proposal](#proposal)\n  - [For Core Master Images](#for-core-master-images)\n    - [\u003ca href=\"https://github.com/kubernetes/kubernetes/blob/caf9d94d697ce327e0c1c3dee71a1f06a6fc918e/build/root/Makefile#L419\"\u003eBash Release\u003c/a\u003e](#bash-release)\n    - [\u003ca href=\"https://github.com/kubernetes/kubernetes/blob/caf9d94d697ce327e0c1c3dee71a1f06a6fc918e/build/root/Makefile#L604\"\u003eBazel Release\u003c/a\u003e](#bazel-release)\n    - [\u003ca href=\"https://github.com/kubernetes/test-infra/tree/master/kubetest\"\u003eTest Release\u003c/a\u003e](#test-release)\n  - [Solution](#solution)\n    - [Notifications to Cloud Providers](#notifications-to-cloud-providers)\n  - [For Generic Add-On Images](#for-generic-add-on-images)\n    - [Example](#example)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n  - [Status Updates](#status-updates)\n  - [Further work](#further-work)\n  - [Rebased Images](#rebased-images)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThe effort of rebasing the k8s images to distroless/static is aimed at making the k8s images thinner, safer and less vulnerable. The scope is not only improving the core containers but will cover the master and node addons which have their own release process. As for the core containers, this effort is targeting the v1.15 release.\n\n## Motivation\n\nRebasing the k8s images to distroless/static can make the images thinner, safer and less vulnerable.\n\nMeanwhile, it will drastically reduce churn on the total number of k8s images versions. Due to the fact that many images are based on debian base and a vulnerability in debian base (a couple times a month) will result in rebuilding every image, changing the image from debian base to distroless/static can reduce the total number of k8s image versions.\n\nWhat's more, it reduces the burden of managing and maintaining multiple k8s images from the security (e.g. CVE), compatibility and build process concerns.\n\n### Goals\n\nUse image gcr.io/distroless/static:latest as the only base image for the following kubernetes images\n\n- Images based FROM scratch\n- Images based on debian/alpine and only for the purpose of redirecting logs with shell.\n- Images based on k8s.gcr.io/debian-base due to previous rebasing from busybox.\n\nHelp the community and contributors better understanding and maintaining the images.\n\n- Set up the policy that only `distroless/static` and `k8s.gcr.io/debian-base` are used (as the base image) for the images hosted in the official k8s.gcr.io image repository. And if the image is based on debian-base, it should be documented in the exception list.\n- Improve the presubmit prow test to guarantee that the upcoming k8s/kubernetes PRs won't introduce dependencies that distroless/static doesn't support.\n- Document the base image list for important kubernetes components, including both core containers and important add-ons. Also, document the exception list (unable to base on distroless).\n\n\n### Non-Goals\n\n- Do not change Images based on debian/alpine that requires fluentd (e.g. hyperkube).\n- Do not change images that have hard dependencies on non-static binaries.\n\n## Background\n\nThis section discusses how the goal and scope are determined due to the reality. It also contains the real use cases.\n\n### Kubernetes Images\n\nKubernetes not only runs images in the containers, but its components themselves are running and deploying as images. Each component image can be built from different base images.\n\nCurrently, kubernetes uses three main types of base images to build their components.\n\n#### Type 1 FROM Scratch\n\nThis docker image is based “FROM scratch” and doesn’t have external dependencies. The original motivation of using “FROM scratch” is to keep the image extremely thin and only contain what a static binaries need. However, caveats are found when running the go static binaries due to some missing non-binary dependencies like ca-certificates, resolv.conf, hosts, and nsswitch (see [issue/69195](https://github.com/kubernetes/kubernetes/issues/69195)).\n\n#### Type 2 Debian Based Images\n\nAn image can be based from Debian due to different reasons.\n\n- One big reason is that the image needs to use shell to redirect the glog. This base image now is an overkill because K8s 1.13 can support using klog which accepts a --log-file flag to point to the log path directly. (Historically images doing this mostly relied on busybox or alpine. Some recent change has migrated off those to debian-base. [PR/70245](https://github.com/kubernetes/kubernetes/pull/70245/files))\n- Another reason of using debian is from the CVE concerns. Those images are originally rebased from busybox to debian for better CVE feeds and management (See [PR/70245](https://github.com/kubernetes/kubernetes/pull/70245)).\n- A third type of images uses debian for certain external dependencies.\n\n#### Type 3 Alpine Based Images\n\nThe reasons for images based on alpine are similar to the ones on debian. Debian is more widely used due to previous “Establish base image policy for system containers” effort (see [issue/40248](https://github.com/kubernetes/kubernetes/issues/40248)).\n\n### Distroless and Previous Work\n\n\"Distroless images contain only your application and its runtime dependencies. They do not contain package managers, shells or any other programs you would expect to find in a standard Linux distribution.”  (See [distroless/README](https://github.com/GoogleContainerTools/distroless))\nDistroless supports the dependencies where “FROM scratch” misses and more light-weighted than debian or alpine. Meanwhile, distroless “improves the signal to noise of scanners (e.g. CVE) and reduces the burden of establishing provenance to just what you need.”(from [distroless/README](https://github.com/GoogleContainerTools/distroless))\n\nUsing Distroless/static as a common image base is originally proposed as an exploration area in [the base image policy for system containers](https://github.com/kubernetes/kubernetes/issues/40248). Tim(tallclair@) has driven the effort on defining and establishing the base image policy (main changes):\n\n* Add Alpine iptable as base image for kube-proxy. Previously kube-proxy is based on debian iptable image.  (This direction is scrapped, see [issue/39696](https://github.com/kubernetes/kubernetes/issues/39696) for details)\n* Rebase busybox images to debian-base.\n* Rebase certain alpine images to debian-base\n\nThe distroless/static solution is filed separately in [issue/70249](https://github.com/kubernetes/kubernetes/issues/70249). This kep, as a more up-to-date version, is slightly different than the original issue.\n\n## Proposal\n\nThe approaches to rebase different containers can vary significantly due to the function of the containers, the cloud-providers’ release workflows, and legacy reasons (repo migration plans, retirement plans, etc). This section will discuss 4 main types of image rebasing strategies, and this should cover the majority of the kubernetes containers.\n\n1. The images are built via bazel. In such case, we will update the bazel BUILD rule to switch to the base image to distroless/static. This method applies for the core containers like kube-apiserver, kube-controller-manager, cloud-controller-manager, kube-scheduler. (See detailed solution in [Core Master Images](##for-core-master-images))\n2. The images have dependencies that are not supported by distroless/static. One typical example is the usage of shell. Previously, shell is widely used for redirecting glog output to a certain directory. This use case is no longer needed since we've switched from glog to klog which can accept a flag to specify the log output path. A generic approach is: Remove the dependencies that distroless/static doesn't support and then rebase the images to distroless (e.g. [issue/1787](https://github.com/kubernetes/autoscaler/issues/1787). Meanwhile, we limit the introduction of new dependencies. (e.g. [pr/74690](https://github.com/kubernetes/kubernetes/pull/74690#discussion_r266037189))\n3. Images based \"FROM scratch\" is safe to switch to distroless/static directly.\n4. Images from kubernetes incubator won't be changed directly by this KEP and the release plan is not estimated here. We notify the project OWNERs and we defer to the OWNERs on whether or not those images should be updated.\n\n### For Core Master Images\n\nThe core master images includes `kube-apiserver`, `kube-controller-manager`, `kube-scheduler` and `kube-proxy`. For `kube-proxy`, it is based on debian-iptable which distroless/static doesn't support iptables yet. Thus, kube-proxy won't be changed.\n\nCurrently, there are **three** different workflows to build the core master images and which workflow to use  is determined by each **cloud provider**.\n\n#### [Bash Release](https://github.com/kubernetes/kubernetes/blob/caf9d94d697ce327e0c1c3dee71a1f06a6fc918e/build/root/Makefile#L419)\n\nRun `make release` under kubernetes repo. This approach is most commonly used and it uses the bash scripts (See [build/release.sh](https://github.com/kubernetes/kubernetes/blob/caf9d94d697ce327e0c1c3dee71a1f06a6fc918e/build/release.sh#L36) for details) to build the images. In this workflow, the base image is specified in the [`build/common.sh`](https://github.com/kubernetes/kubernetes/blob/f26048ecb1c7b6fb67c2e7c7c96070d7a1743d86/build/common.sh#L96).\n\n#### [Bazel Release](https://github.com/kubernetes/kubernetes/blob/caf9d94d697ce327e0c1c3dee71a1f06a6fc918e/build/root/Makefile#L604)\n\nRun `make bazel-release` under kubernetes repo. This approach uses bazel to build the image artifact based on this [BUILD](https://github.com/kubernetes/kubernetes/blob/caf9d94d697ce327e0c1c3dee71a1f06a6fc918e/build/release-tars/BUILD) rule (More details in [bazel.bzl/release-filegroup](https://github.com/kubernetes/repo-infra/blob/4528e18f5d62a2a5172f76d198738d85d4d04734/defs/build.bzl#L134)) . In this workflow, the base image is specified in the [build/BUILD](https://github.com/kubernetes/kubernetes/blob/3fd6f97f55d51c01df3c01c7ffbb2834c25d9900/build/BUILD#L31) rule.\n\n#### [Test Release](https://github.com/kubernetes/test-infra/tree/master/kubetest)\n\nRun `kubetest` or `hack/e2e`. See details in the [test-infra repo](https://github.com/kubernetes/test-infra/tree/master/kubetest). This approach is recommended for development testing and is broadly used by contributors. However, this approach is under a test env and it uses different config than the two official workflows as described above. In this workflow, the base image is specified to use k8s.gcr.io/pause:3.1.\n\n### Solution\n\nThis KEP is expected to rebase images for all three workflows. This requires each cloud provider team to be involved in the manifest updates and release workflow testing part (See the graph below). Before we switch the base images to `distroless/static`, each cloud provider team should make sure their manifest config is updated so that the **command doesn’t require shell to run the executable binaries and no log redirection is involved in the command**. Otherwise rebasing images to distroless will **break** the core containers running in the cluster master VMs. The test release should also be updated to `distroless/static` so as we can guarantee further changes wouldn’t be able to add unexpected dependencies (otherwise, they will fail the e2e tests in the github prow test stage).\n\n![Rebase Core Master](RebaseCoreMaster.png?raw=true \"Rebase Core Master\")\n\n#### Notifications to Cloud Providers\n\n- For log redirection, please use flag [`log-file`](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver) (e.g. `--log-file=/var/log/kube-controller-manager.log`) and also disable standard output (e.g. `--logtostderr=false`)\n- When removing the shell from manifest command, please also update the parameter format to exec form.` [“executable”, “param1, “param2”]`\n- See example in [PR/75624](https://github.com/kubernetes/kubernetes/pull/75624)\n- Detailed timelines about switching to the distroless/static will be announced later on. Please make sure manifest change is well tested in the release workflow (as shown in the right blue part).\n\n### For Generic Add-On Images\n\nIt more or less depends on add-on OWNERs’ judge on whether/how the add-on images should be rebased. The below progress is what we proposed to the OWNERs. This should apply for most use cases.\n\n1. (If the images depends on a k8s version that is earlier than v1.13) Sync up with current k8s head. Since kubernetes 1.13,oss kubernetes no longer uses glog which requires shell to redirect the log file. Instead, k8s is using klog which accepts a log path flag. This sync-up is necessary to remove log redirection.\n2. (If the images use glog) Replace the glog to klog inside the add-on files.\n3. Update the base image to distroless and remove distroless-preinstalled packages like ca-certificate.\n4. (If necessary) Update the container upstart command to avoid using bash command. (For log redirection, see examples in the *For Core Master Images* section).\n5. If bash scripts can’t be easily removed, document the container as exception in **[this list](https://github.com/kubernetes/sig-release/blob/master/release-engineering/baseimage-exception-list.md)**\n6. After the above steps are done, require release engineers' help on monitoring the performance.\n\n#### Example\n\n[ingress-gce/fuzzer](https://github.com/kubernetes/ingress-gce/blob/64eee7e3521680057b071d5e9bebaa215086a4bc/Dockerfile.fuzzer) was based on alpine and can't be switched to distroless directly due to the fact that it needs shell to redirect the glog file. To allow the images to be based on distroless (which doesn't contain shell), we firstly need to remove the dependency on the shell (use klog instead of glog), and then rebase the image. (Related PR [pr/682](https://github.com/kubernetes/ingress-gce/pull/682), [pr/666](https://github.com/kubernetes/ingress-gce/pull/666))\n\n\n## Graduation Criteria\n\nThis KEP is targeted at v1.15 release. The full list of images switched to distroless/static will be updated later on.\n\n## Implementation History\n\n### Status Updates\n- Rebased the [following images](#rebased-images) to `gcr.io/distroless/static:latest` or `k8s.gcr.io/debian-base:v1.0.0`.\n- Investigated [these images](https://github.com/kubernetes/sig-release/blob/master/release-engineering/baseimage-exception-list.md) as exceptions (can't based on distroless).\n- Triaged and fixed the following issues which blocked rebasing images:\n  * [Avoid log truncation due to log-rotation delays](https://github.com/kubernetes/klog/issues/55#issuecomment-481032528)\n  * [Log duplication with --log-file](https://github.com/kubernetes/klog/pull/65)\n  * [Fix MakeFile push workflow for metrics-server](https://github.com/kubernetes-incubator/metrics-server/pull/259)\n\n### Further work\n- Triage klog for the performance regression on core master containers:\n  * Affected images: kube-controller-manager, kube-scheduler, kube-apiserver\n  * Blocked PRs:\n     - [Update manifests to use klog --log-file](https://github.com/kubernetes/kubernetes/pull/78466)\n     - [Rebase core master images for both bazel and bash release](https://github.com/kubernetes/kubernetes/pull/75306)\n- Avoid using exec in kube-controller-manager for flexvolume.\n\n### Rebased Images\n\n| Component Name        |   on Master/Node         |  Previous Image --\u003e Current Image   |      Image      |   Code Complete  |   Release Complete |    Contact        |\n|     addon-resize      |  Master + Node           |        Busybox  --\u003e distroless      |  k8s.gcr.io/addon-resizer:1.8.5 |      Done        |   Done           |    @bskiba @yuwenma |\n|     cluster-proportional-autoscaler  |  Master + Node   |        scratch  --\u003e distroless      |  k8s.gcr.io/cluster-proportional-autoscaler-arm:v1.6.0 |  Done | Done |  @yuwenma @MrHohn |\n|     cluster-proportional-vertical-autoscaler  |  Master + Node   |        scratch  --\u003e distroless      |  k8s.gcr.io/cpvpa-amd64:v0.7.1 |  Done | Done |  @yuwenma @MrHohn |\n|     event-exporter  |  Master + Node   |        debian-base  --\u003e distroless      |  k8s.gcr.io/event-exporter:v0.2.5 |  Done | Done |  @x13n @yuwenma |\n|     node-termination-handler  |  Master + Node   |        alpine  --\u003e distroless      |  [k8s.gcr.io/gke-node-termination-handler](https://pantheon.corp.google.com/gcr/images/google-containers/GLOBAL/gke-node-termination-handler@sha256:aca12d17b222dfed755e28a44d92721e477915fb73211d0a0f8925a1fa847cca/details?tab=info) |  Done | Done |  @yuwenma |\n|     metadata-proxy  |  Master + Node   |        scratch  --\u003e distroless       |  k8s.gcr.io/metadata-proxy:v0.1.12 |  Done | Done |  @dekkagaijin @yuwenma |\n|     metrics-server  |  Master + Node   |        busybox  --\u003e distroless       |  k8s.gcr.io/metrics-server:v0.3.3 |  Done | Done |  @yuwenma @kawych  |\n|     prometheus-to-sd  |  Master + Node   |        debian-base  --\u003e distroless       |  k8s.gcr.io/metrics-server:v0.5.2 |  Done | Done |  @loburm   |\n|     ip-masq-agent  |  Master + Node   |         busybox  --\u003e debian-iptables       |  k8s.gcr.io/ip-masq-agent:v2.4.1 |  Done | Done |  @BenTheElder @yuwenma   |\n|     slo-monitor  |  Master   |        alpine  --\u003e distroless       |  k8s.gcr.io/slo-monitor:0.11.2 |  Done | Done |  @yuwenma   |\n|     kubelet-to-gcm   |  Master   |        scratch  --\u003e distroless       |  k8s.gcr.io/kubelet-to-gcm:v1.2.11 |  Done | wait for next release |  @yuwenma |\n|     etcd-version-monitor   |  Master   |        scratch  --\u003e distroless       |  k8s.gcr.io/etcd-version-monitor:v0.1.3 |  Done | Done |  @yuwenma |\n|     etcd-empty-dir-cleanup   |  Master   |        busybox  --\u003e distroless       |  k8s.gcr.io/etcd-empty-dir-cleanup:3.3.10.1 |  Done | Done |  @yuwenma |\n|     etcd |  Master   |        busybox  --\u003e distroless       |  k8s.gcr.io/etcd:3.3.10-1 |  Done | Done |  @yuwenma |\n|     defaultbackend  |  Master + Node   | scratch  --\u003e distroless  | Wait for next release  |  Done | targeting v1.16 | @rramkumar1 @yuwenma  |\n|     fuzzer   |  Master + Node   | alpine  --\u003e distroless  | Wait for next release  |  Done | targeting v1.16 |  @rramkumar1 @yuwenma   |\n|     ingress-gce-glbc  |  Master + Node   | alpine  --\u003e distroless  | Wait for next release |  Done | targeting v1.16 | @rramkumar1 @yuwenma    |\n|     k8s-dns-kube-dns  |  Master + Node   | alpine  --\u003e debian-base  | k8s.gcr.io/k8s-dns-kube-dns:1.15.3 |  Done | Done | @yuwenma @prameshj  |\n|     k8s-dns-sidecar  |  Master + Node   | alpine  --\u003e debian-base  | k8s.gcr.io/k8s-dns-sidecar:1.15.3 |  Done |Done  | @yuwenma @prameshj  |\n|     k8s-dns-dnsmasq-nanny |  Master + Node   | alpine  --\u003e debian-base  | k8s.gcr.io/k8s-dns-dnsmasq-nanny:1.15.3  |  Done | Done   | @yuwenma @prameshj  |\n|     k8s-dns-node-cache |  Node   | debian:stable-slim  --\u003e debian-base  | k8s.gcr.io/k8s-dns-node-cache:1.15.3 |  Done | Done | @yuwenma @prameshj  |\n|     cluster-autoscaler |  Master   | debian-base  --\u003e distroless  | k8s.gcr.io/cluster-autoscaler:v1.16.0 |  Done | Done | @losipiuk  |\n"
+  },
+  {
+    "id": "9e42f7428fee6cfde242ce2229fc2b4c",
+    "title": "Release Notes Improvements",
+    "authors": ["@jeefy"],
+    "owningSig": "sig-release",
+    "participatingSigs": ["sig-contributor-experience", "sig-docs"],
+    "reviewers": ["@spiffxp", "@marpaia"],
+    "approvers": ["@calebamiles", "@tpepper", "@justaugustus"],
+    "editor": "TBD",
+    "creationDate": "2019-03-31",
+    "lastUpdated": "2019-03-31",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Release Notes Improvements\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Consolidation and Clean-up](#consolidation-and-clean-up)\n    - [Updating Anago](#updating-anago)\n    - [Release Notes Website](#release-notes-website)\n  - [Automation](#automation)\n    - [Build additional labels](#build-additional-labels)\n    - [Automated release notes](#automated-release-notes)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Graduation Criteria](#graduation-criteria)\n- [Infrastructure Needed](#infrastructure-needed)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis document describes a new release notes process as well as a new site for end-users to\nbetter consume the generated data. While this change would only affect the release-notes\nteam, this is a visible change large enough to warrant a KEP.\n\n## Motivation\n\nThe current release notes process is fraught with inefficiencies that should be streamlined.\n\n- There are two different ways to generate release notes:\n[relnotes](https://github.com/kubernetes/release/blob/master/relnotes) and\n[release-notes](https://github.com/kubernetes/release/tree/master/cmd/release-notes).\n  - Because of this there's duplicate effort maintaining separate tools.\n- We are currently shipping entire changelogs within markdown for each release.\n- PRs are frequently categorized within several SIGs, which the release notes team has to\n  distill down for the sake of brevity before the release is actually cut.\n- An end-user may not need to see _every single change_ that occurred within a release.\n\nThere is room for improvement in the generation as well as the consumption of release notes.\nIn order to make the process more sustainable, and improve end-user experience, we should put\neffort into automation as well as better ways to consume release notes.\n\n### Goals\n\nAs this is a somewhat drastic departure from the current process, this should be a two-phased\napproach:\n\n- Consolidation and Clean-up\n  - Update `anago` to use `release-notes`\n  - Replace the \"Detailed Bug Fixes And Changes\" section in the release notes with a new\n    website that allows user to filter and search.\n  - Generate a consumable JSON file via `release-notes` as part of the Release Team duties\n  - Identify and stop tracking any \"External Dependencies\" that a vanilla Kubernetes install\n    does not rely on. (eg. Things in `cluster/addons`)\n- Automation\n  - Build additional labels to classify:\n    - API Changes\n    - Deprecations\n    - Urgent Upgrade Notes\n  - Capture release notes automatically via GitHub PR webhooks.\n    - Use milestones to capture \"Known Issues\" at time of release notes generation\n\n### Non-Goals\n\nThis is scoped solely around generating release notes for the main Kubernetes releases.\n\n## Proposal\n\nAs stated above, this effort should be split into two phases.\n\n### Consolidation and Clean-up\n\n#### Updating Anago\n\nCurrently, `anago` uses a \n[different tool](https://github.com/kubernetes/release/blob/master/relnotes) to generate new\nrelease notes with every release save for main 1.x.0 releases. We need to ensure that the\n`release-notes` tool can generate the same output as the `relnotes` tool to ensure consistency.\n\n#### Release Notes Website\n\nSome progress has already been made (as a [POC Website](https://k8s-relnotes.netlify.com/)) to\ndrum up interest. We need to move the current codebase out of a\n[personal repo](https://github.com/jeefy/relnotes) and into an official repo.\n\nThe `release-notes` tool is already capable of outputting a JSON format of the release notes,\nwhich is what the current POC is consuming.\n\n### Automation\n\n#### Build additional labels\n\nOnce we can reliably generate a changelog, we should then strive to automate classifying the other\ncomponents of release notes. The notable sections are:\n\n- API Changes (release/api_change)\n- Deprecations (release/deprecation)\n- Urgent Upgrade Notes (release/urgent)\n\nOn top of advertising these new labels, the release notes team should actively monitor and apply\nthese labels to ensure less manual classification.\n\n#### Automated release notes\n\nIt should be possible to completely automate the generation and publishing of release notes. By\nbuilding a Knative pipeline that is fed from GitHub PR events, we could have it grab, format,\nand commit new entries into the release notes website. As we don't want these notes to be\npublished before a release goes live, we would create a \"draft\" flag in the `release-notes` JSON\nschema.\n\nOnce a release is ready to be cut, the release notes team would then flip all the 1.x notes that\nhave been collected to not be drafts. This would be done by cloning down the release notes\nwebsite, and running the `release-notes` tool over the JSON and committing the new output.\n\nExample:\n\n```bash\nrelease-notes -i relnotes.json -p 1.15.0\n```\n\n### Risks and Mitigations\n\nAutomation inevitably fails. Once the full automated release notes process has been implemented,\nwe will need a means to monitor it. The fallback would be to manually generate and commit a JSON\nfile to the release notes website.\n\n## Design Details\n\n### Graduation Criteria\n\nWhile this isn't directed at any single release, the goal is to phase the full implementation in\nover multiple releases. Ultimately, this KEP would graduate once we have a dedicated release notes\nwebsite that is automatically updated with minimal human interaction.\n\n## Infrastructure Needed\n\nA GitHub repo will need to be setup to host code for both the `release-notes` tool as well as the\nnew release notes website. Said repo will also need to be integrated with Netlify for hosting. An\ninitial design idea to power the automatic generation of release notes was to use a Knative pipeline.\nThis would require a Kubernetes cluster to run on, as well as a GitHub token and webhook registration.\nLastly, a DNS entry will be needed to point to the Netlify site (proposal: relnotes.k8s.io)\n"
+  },
+  {
+    "id": "878635cf71802e1a5b73c4f8ccf7f6d0",
+    "title": "Image Promoter",
+    "authors": ["@javier-b-perez"],
+    "owningSig": "sig-release",
+    "participatingSigs": ["wg-k8s-infra"],
+    "reviewers": ["@AishSundar", "@BenTheElder", "@dims", "@listx"],
+    "approvers": ["@thockin"],
+    "editor": "",
+    "creationDate": "2018-09-05",
+    "lastUpdated": "2018-11-14",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Image Promoter\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n- [Proposal](#proposal)\n  - [Staging Container Registry](#staging-container-registry)\n  - [Production Container Registry](#production-container-registry)\n  - [Promotion Process](#promotion-process)\n- [Graduation Criteria](#graduation-criteria)\n- [Infrastructure Needed](#infrastructure-needed)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nFor security reasons, we cannot allow everyone to publish container images into the official kubernetes container registry. This is why we need a process that allows us to review who built an image and who approved it to be shown in the official channels.\n\n\n## Motivation\n\nThere are multiple reasons why we should have a process to publish container images in place:\n\n* We cannot allow all community members to publish images into the official kubernetes container registry.\n* We should restrict who can push images to a small set of members and some systems accounts for automation.\n* We can run scans and tests on the images before we publish them into the official kubernetes container registry.\n* The process to publish into an official channel shouldn't be hard or long to follow. We don’t want to block developers or releases.\n\n### Goals\n\n1. Define a process for publishing container images into an official GCR through a code review process and automated promotion from GCR staging environment.\n1. Allow the community to own and manage the project registries.\n\n## Proposal\n\nFollowing the *GitOps* idea, the proposal is to use a code review process to approve publishing container images into official distribution channels.\n\nThis requires two GCR registries:\n\n* Staging: temporary container registry to share container images for testing and scanning.\n* Production or *official*: GCR used to host all the approved container images by the community.\n\n### Staging Container Registry\n\nThis temporary storage allows to have a public place where to pull images and run qualification tests or vulnerability scans on the images before pushing them to the *official* container registry.\n\nEach project/subproject in the community, will require at least one member of their community to have push access to the staging area.\n\n### Production Container Registry\n\nA restricted set of members can have push access to override any tool or process if necessary.\nIdeally we only push images that have been approved by the owners of the production container registry, following the promotion process.\n\n### Promotion Process\n\n1. Maintainer create a container image and push it into *staging* GCR.\n1. Maintainer creates a PR in GitHub to add the new image into the *official* container registry.\n1. Once owners of the official container registry approve the change and merge it into the master branch, the promoter tool will automatically copy the container image(s) from *staging* into *official* container registry.\n\nIf the infrastructure support it, the promoter tool could sign container images when pushing to the official container registry.\n\n![Promote process](promote-process.jpg?raw=true \"Promote process\")\n\nIn the future, we could add more information into the context of the PR like the vulnerability scan and test results of the container image.\n\n## Graduation Criteria\n\nWe will know we are done when we have:\n\n* User guide for developers/maintainers: how to build? how to promote?\n* User guide for owners: review and approve PR, how to push images?\n* A repository to host the manifest file.\n* Initial set of repository's owners who can approve changes.\n* Criteria to grant or remove access to staging and production container registries.\n* A tool that automatically copy approved images into official channels.\n\n## Infrastructure Needed\n\n* Two GCP projects with GCR enabled.\n  * One project should have GCB enabled to run the promotion tool in it.\n* Repository to host the manifest for promotions.\n\n"
+  },
+  {
+    "id": "69c6c8dc6c12c9a6f9d89a3e3a51ebe1",
+    "title": "Scheduling Framework",
+    "authors": ["@bsalamat", "@misterikkit"],
+    "owningSig": "sig-scheduling",
+    "participatingSigs": null,
+    "reviewers": ["@huang-wei", "@k82cn", "@ravisantoshgudimetla"],
+    "approvers": ["@k82cn"],
+    "editor": "TBD",
+    "creationDate": "2018-04-09",
+    "lastUpdated": "2019-04-30",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": [
+      "https://github.com/kubernetes/community/blob/master/contributors/design-proposals/scheduling/scheduling-framework.md"
+    ],
+    "supersededBy": null,
+    "markdown": "# Scheduling Framework\n\n\u003c!-- toc --\u003e\n- [SUMMARY](#summary)\n- [MOTIVATION](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [PROPOSAL](#proposal)\n  - [Scheduling Cycle \u0026amp; Binding Cycle](#scheduling-cycle--binding-cycle)\n  - [Extension points](#extension-points)\n    - [Queue sort](#queue-sort)\n    - [Pre-filter](#pre-filter)\n    - [Filter](#filter)\n    - [Pre-Score](#pre-score)\n    - [Scoring](#scoring)\n    - [Reserve](#reserve)\n    - [Permit](#permit)\n    - [Pre-bind](#pre-bind)\n    - [Bind](#bind)\n    - [Post-bind](#post-bind)\n    - [Un-reserve](#un-reserve)\n  - [Plugin API](#plugin-api)\n    - [CycleState](#cyclestate)\n    - [FrameworkHandle](#frameworkhandle)\n    - [Plugin Registration](#plugin-registration)\n  - [Plugin Lifecycle](#plugin-lifecycle)\n    - [Initialization](#initialization)\n    - [Concurrency](#concurrency)\n  - [Configuring Plugins](#configuring-plugins)\n    - [Enable/Disable](#enabledisable)\n    - [Change Evaluation Order](#change-evaluation-order)\n    - [Optional Args](#optional-args)\n    - [Backward compatibility](#backward-compatibility)\n  - [Interactions with Cluster Autoscaler](#interactions-with-cluster-autoscaler)\n- [USE CASES](#use-cases)\n  - [Coscheduling](#coscheduling)\n  - [Dynamic Resource Binding](#dynamic-resource-binding)\n  - [Custom Scheduler Plugins (out of tree)](#custom-scheduler-plugins-out-of-tree)\n- [TEST PLANS](#test-plans)\n- [GRADUATION CRITERIA](#graduation-criteria)\n- [IMPLEMENTATION HISTORY](#implementation-history)\n\u003c!-- /toc --\u003e\n\n# SUMMARY\n\nThis document describes the Kubernetes Scheduling Framework. The scheduling\nframework is a new set of \"plugin\" APIs being added to the existing Kubernetes\nScheduler. Plugins are compiled into the scheduler, and these APIs allow many\nscheduling features to be implemented as plugins, while keeping the scheduling\n\"core\" simple and maintainable.\n\n*Note: Previous versions of this document proposed replacing the existing\nscheduler with a new implementation.*\n\n# MOTIVATION\n\nMany features are being added to the Kubernetes Scheduler. They keep making the\ncode larger and the logic more complex. A more complex scheduler is harder to\nmaintain, its bugs are harder to find and fix, and those users running a custom\nscheduler have a hard time catching up and integrating new changes. The current\nKubernetes scheduler provides [webhooks to extend][] its functionality. However,\nthese are limited in a few ways:\n\n[webhooks to extend]: https://github.com/kubernetes/community/blob/master/contributors/design-proposals/scheduling/scheduler_extender.md\n\n1.  The number of extension points are limited: \"Filter\" extenders are called\n    after default predicate functions. \"Prioritize\" extenders are called after\n    default priority functions. \"Preempt\" extenders are called after running\n    default preemption mechanism. \"Bind\" verb of the extenders are used to bind\n    a Pod. Only one of the extenders can be a binding extender, and that\n    extender performs binding instead of the scheduler. Extenders cannot be\n    invoked at other points, for example, they cannot be called before running\n    predicate functions.\n1.  Every call to the extenders involves marshaling and unmarshalling JSON.\n    Calling a webhook (HTTP request) is also slower than calling native\n    functions.\n1.  It is hard to inform an extender that scheduler has aborted scheduling of a\n    Pod. For example, if an extender provisions a cluster resource and scheduler\n    contacts the extender and asks it to provision an instance of the resource\n    for the pod being scheduled and then scheduler faces errors scheduling the\n    pod and decides to abort the scheduling, it will be hard to communicate the\n    error with the extender and ask it to undo the provisioning of the resource.\n1.  Since current extenders run as a separate process, they cannot use\n    scheduler's cache. They must either build their own cache from the API\n    server or process only the information they receive from the default\n    scheduler.\n\nThe above limitations hinder building high performance and versatile scheduler\nfeatures. We would ideally like to have an extension mechanism that is fast\nenough to allow existing features to be converted into plugins, such as\npredicate and priority functions. Such plugins will be compiled into the\nscheduler binary. Additionally, authors of custom schedulers can compile a\ncustom scheduler using (unmodified) scheduler code and their own plugins.\n\n## Goals\n\n-   Make scheduler more extendable.\n-   Make scheduler core simpler by moving some of its features to plugins.\n-   Propose extension points in the framework.\n-   Propose a mechanism to receive plugin results and continue or abort based on\n    the received results.\n-   Propose a mechanism to handle errors and communicate them with plugins.\n\n## Non-Goals\n\n-   Solve all scheduler limitations, although we would like to ensure that the\n    new framework allows us to address known limitations in the future.\n-   Provide implementation details of plugins and call-back functions, such as\n    all of their arguments and return values.\n\n# PROPOSAL\n\nThe Scheduling Framework defines new extension points and Go APIs in the\nKubernetes Scheduler for use by \"plugins\". Plugins add scheduling behaviors to\nthe scheduler, and are included at compile time. The scheduler's ComponentConfig\nwill allow plugins to be enabled, disabled, and reordered. Custom schedulers can\nwrite their plugins \"[out-of-tree](#custom-scheduler-plugins-out-of-tree)\" and\ncompile a scheduler binary with their own plugins included.\n\n## Scheduling Cycle \u0026 Binding Cycle\n\nEach attempt to schedule one pod is split into two phases, the **scheduling\ncycle** and the **binding cycle**. The scheduling cycle selects a node for the\npod, and the binding cycle applies that decision to the cluster. Together, a\nscheduling cycle and binding cycle are referred to as a \"scheduling context\".\nScheduling cycles are run serially, while binding cycles may run concurrently.\n(See [Concurrency](#concurrency))\n\nA scheduling cycle or binding cycle can be aborted if the pod is determined to\nbe unschedulable or if there is an internal error. The pod will be returned to\nthe queue and retried. If a binding cycle is aborted, it will trigger\n[Un-reserve](#un-reserve) plugins.\n\n## Extension points\n\nThe following picture shows the scheduling context of a pod and the extension\npoints that the scheduling framework exposes. In this picture \"Filter\" is\nequivalent to \"Predicate\" and \"Scoring\" is equivalent to \"Priority function\".\nPlugins are registered to be called at one or more of these extension points. In\nthe following sections we describe each extension point in the same order they\nare called.\n\nOne plugin may register at multiple extension points to perform more complex or\nstateful tasks.\n\n![image](20200125-scheduling-framework-extensions.png)\n\n### Queue sort\n\nThese plugins are used to sort pods in the scheduling queue. A queue sort plugin\nessentially will provide a \"less(pod1, pod2)\" function. Only one queue sort\nplugin may be enabled at a time.\n\n### Pre-filter\n\nThese plugins are used to pre-process info about the pod, or to check certain\nconditions that the cluster or the pod must meet. A pre-filter plugin should implement\na PreFilter function, if PreFilter returns an error, the scheduling cycle is aborted.\nNote that PreFilter is called once in each scheduling cycle.\n\nA Pre-filter plugin can implement the optional `PreFilterExtensions` interface which\ndefine **AddPod** and **RemovePod** methods to incrementally modify its pre-processed info.\nThe framework guarantees that those functions will only be called after PreFilter, possibly \non a cloned CycleState, and may call those functions more than once before calling Filter on \na specific node.\n\n\n### Filter\n\nThese plugins are used to filter out nodes that cannot run the Pod. For each\nnode, the scheduler will call filter plugins in their configured order. If any\nfilter plugin marks the node as infeasible, the remaining plugins will not be\ncalled for that node. Nodes may be evaluated concurrently, and Filter may be called\nmore than once in the same scheduling cycle.\n\n### Pre-Score\n\n**Notice: `Pre-Score` is available since v1alpha2, and it's known as `Post-Filter` before this version.**\n\nThis is an informational extension point for performing pre-scoring work. Plugins will be called with a list of\nnodes that passed the filtering phase. A plugin may use this data to update internal state or to generate logs/metrics.\n\n### Scoring\n\nThese plugins have two phases:\n\n1. The first phase is called \"score\" which is used to rank nodes that have passed\nthe filtering phase. The scheduler will call `Score` of each scoring plugin for\neach node.\n2. The second phase is \"normalize scoring\" which is used to modify scores before\nthe scheduler computes a final ranking of Nodes, and each score plugin receives\nscores given by the same plugin to all nodes in \"normalize scoring\" phase.\n`NormalizeScore` is called once per plugin per scheduling cycle right after\n\"score\" phase. Note that `NormalizeScore` is optional, and can be provided\nby implementing the `ScoreExtensions` interface.\n\nThe output of a score plugin must be an integer in range of\n**[MinNodeScore, MaxNodeScore]**. if not, the scheduling cycle is aborted.\nThis is the output after running the optional NormalizeScore function of the\nplugin. If NormalizeScore is not provided, the output of Score must be in this range.\nAfter the optional NormalizeScore, the scheduler will combine node scores from all\nplugins according to the configured plugin weights.\n\nFor example, suppose a plugin `BlinkingLightScorer` ranks Nodes based on how\nmany blinking lights they have.\n\n```go\nfunc (*BlinkingLightScorer) Score(state *CycleState, _ *v1.Pod, nodeName string) (int, *Status) {\n   return getBlinkingLightCount(nodeName)\n}\n```\n\nHowever, the maximum count of blinking lights may be small compared to\n`MaxNodeScore`. To fix this, `BlinkingLightScorer` should also implement `NormalizeScore`.\n\n```go\nfunc (*BlinkingLightScorer) NormalizeScore(state *CycleState, _ *v1.Pod, nodeScores NodeScoreList) *Status {\n   highest := 0\n   for _, nodeScore := range nodeScores {\n      highest = max(highest, nodeScore.Score)\n   }\n   for i, nodeScore := range nodeScores {\n      nodeScores[i].Score = nodeScore.Score*MaxNodeScore/highest\n   }\n   return nil\n}\n```\n\nIf either `Score` or `NormalizeScore` returns an error, the scheduling cycle is aborted.\n\n### Reserve\n\nThis is an informational extension point. Plugins which maintain runtime state\n(aka \"stateful plugins\") should use this extension point to be notified by the\nscheduler when resources on a node are being reserved for a given Pod. This\nhappens before the scheduler actually binds the pod to the Node, and it exists\nto prevent race conditions while the scheduler waits for the bind to succeed.\n\nThis is the last step in a scheduling cycle. Once a pod is in the reserved\nstate, it will either trigger [Un-reserve](#un-reserve) plugins (on failure) or\n[Post-bind](#post-bind) plugins (on success) at the end of the binding cycle.\n\n*Note: This concept used to be referred to as \"assume\".*\n\n### Permit\n\nThese plugins are used to prevent or delay the binding of a Pod. A permit plugin\ncan do one of three things.\n\n1.  **approve** \\\n    Once all permit plugins approve a pod, it is sent for binding.\n\n1.  **deny** \\\n    If any permit plugin denies a pod, it is returned to the scheduling queue.\n    This will trigger [Un-reserve](#un-reserve) plugins.\n\n1.  **wait** (with a timeout) \\\n    If a permit plugin returns \"wait\", then the pod is kept in the permit phase\n    until a [plugin approves it](#frameworkhandle). If a timeout occurs, **wait**\n    becomes **deny** and the pod is returned to the scheduling queue, triggering\n    [un-reserve](#un-reserve) plugins.\n\n**Approving a pod binding**\n\nWhile any plugin can receive the list of reserved pods from the cache and\napprove them (see [`FrameworkHandle`](#frameworkhandle)) we expect only the permit\nplugins to approve binding of reserved Pods that are in \"waiting\" state. Once a\npod is approved, it is sent to the pre-bind phase.\n\n### Pre-bind\n\nThese plugins are used to perform any work required before a pod is bound. For\nexample, a pre-bind plugin may provision a network volume and mount it on the\ntarget node before allowing the pod to run there.\n\nIf any pre-bind plugin returns an error, the pod is [rejected](#un-reserve) and\nreturned to the scheduling queue.\n\n### Bind\n\nThese plugins are used to bind a pod to a Node. Bind plugins will not be called\nuntil all pre-bind plugins have completed. Each bind plugin is called in the\nconfigured order. A bind plugin may choose whether or not to handle the given\nPod. If a bind plugin chooses to handle a Pod, **the remaining bind plugins are\nskipped**.\n\n### Post-bind\n\nThis is an informational extension point. Post-bind plugins are called after a\npod is successfully bound. This is the end of a binding cycle, and can be used\nto clean up associated resources.\n\n### Un-reserve\n\nThis is an informational extension point. If a pod was reserved and then\nrejected in a later phase, then un-reserve plugins will be notified. Un-reserve\nplugins should clean up state associated with the reserved Pod.\n\nPlugins that use this extension point usually should also use\n[Reserve](#reserve).\n\n## Plugin API\n\nThere are two steps to the plugin API. First, plugins must register and get\nconfigured, then they use the extension point interfaces. Extension point\ninterfaces have the following form.\n\n```go\ntype Plugin interface {\n   Name() string\n}\n\ntype QueueSortPlugin interface {\n   Plugin\n   Less(*PodInfo, *PodInfo) bool\n}\n\n\ntype PreFilterPlugin interface {\n   Plugin\n   PreFilter(CycleState, *v1.Pod) *Status\n}\n\n// ...\n```\n\n### CycleState\n\nMost* plugin functions will be called with a `CycleState` argument. A\n`CycleState` represents the current scheduling context.\n\nA `CycleState` will provide APIs for accessing data whose scope is the\ncurrent scheduling context. Because binding cycles may execute concurrently,\nplugins can use the `CycleState` to make sure they are handling the right\nrequest.\n\nThe `CycleState` also provides an API similar to\n[`context.WithValue`](https://godoc.org/context#WithValue) that can be used to\npass data between plugins at different extension points. Multiple plugins can\nshare the state or communicate via this mechanism. The state is preserved only\nduring a single scheduling context. It is worth noting that plugins are assumed\nto be **trusted**. The scheduler does not prevent one plugin from accessing or\nmodifying another plugin's state.\n\n\\* *The only exception is for [queue sort](#queue-sort) plugins.*\n\n**WARNING**: The data available through a `CycleState` is not valid after a\nscheduling context ends, and plugins should not hold references to that data\nlonger than necessary.\n\n### FrameworkHandle\n\nWhile the `CycleState` provides APIs relevant to a single scheduling context,\nthe `FrameworkHandle` provides APIs relevant to the lifetime of a plugin. This\nis how plugins can get a client (`kubernetes.Interface`) and\n`SharedInformerFactory`, or read data from the scheduler's cache of cluster\nstate. The handle will also provide APIs to list and approve or reject\n[waiting pods](#permit).\n\n**WARNING**: `FrameworkHandle` provides access to both the kubernetes API server\nand the scheduler's internal cache. The two are **not guaranteed to be in sync**\nand extreme care should be taken when writing a plugin that uses data from both\nof them.\n\nProviding plugins access to the API server is necessary to implement useful\nfeatures, especially when those features consume object types that the scheduler\ndoes not normally consider. Providing a `SharedInformerFactory` allows plugins\nto share caches safely.\n\n### Plugin Registration\n\nEach plugin must define a constructor and add it to the hard-coded registry. For\nmore information about constructor args, see [Optional Args](#optional-args).\n\nExample:\n\n```go\ntype PluginFactory = func(runtime.Unknown, FrameworkHandle) (Plugin, error)\n\ntype Registry map[string]PluginFactory\n\nfunc NewRegistry() Registry {\n   return Registry{\n      fooplugin.Name: fooplugin.New,\n      barplugin.Name: barplugin.New,\n      // New plugins are registered here.\n   }\n}\n```\n\nIt is also possible to add plugins to a `Registry` object and inject that into a\nscheduler. See [Custom Scheduler Plugins](#custom-scheduler-plugins-out-of-tree)\n\n## Plugin Lifecycle\n\n### Initialization\n\nThere are two steps to plugin initialization. First,\n[plugins are registered](#plugin-registration). Second, the scheduler uses its\nconfiguration to decide which plugins to instantiate. If a plugin registers for\nmultiple extension points, *it is instantiated only once*.\n\nWhen a plugin is instantiated, it is passed [config args](#optional-args) and a\n[`FrameworkHandle`](#frameworkhandle).\n\n### Concurrency\n\nThere are two types of concurrency that plugin writers should consider. A plugin\nmight be invoked several times concurrently when evaluating multiple nodes, and\na plugin may be called concurrently from *different\n[scheduling contexts](#scheduling-cycle--binding-cycle)*.\n\n*Note: Within one scheduling context, each extension point is evaluated\nserially.*\n\nIn the main thread of the scheduler, only one scheduling cycle is processed at a\ntime. Any extension point up to and including [reserve](#reserve) will be\nfinished before the next scheduling cycle begins*. After the reserve phase, the\nbinding cycle is executed asynchronously. This means that a plugin could be\ncalled concurrently from two different scheduling contexts, provided that at\nleast one of the calls is to an extension point after reserve. Stateful plugins\nshould take care to handle these situations.\n\nFinally, [un-reserve](#un-reserve) plugins may be called from either the Permit\nthread or the Bind thread, depending on how the pod was rejected.\n\n\\* *The queue sort extension point is a special case. It is not part of a\nscheduling context and may be called concurrently for many pod pairs.*\n\n![image](20180409-scheduling-framework-threads.png)\n\n## Configuring Plugins\n\nThe scheduler's component configuration will allow for plugins to be enabled,\ndisabled, or otherwise configured. Plugin configuration is separated into two\nparts.\n\n1.  A list of enabled plugins for each extension point (and the order they\n    should run in). If one of these lists is omitted, the default list will be\n    used.\n1.  An optional set of custom plugin arguments for each plugin. Omitting config\n    args for a plugin is equivalent to using the default config for that plugin.\n\nThe plugin configuration is organized by extension points. A plugin that\nregisters with multiple points must be included in each list.\n\n```go\ntype KubeSchedulerConfiguration struct {\n    // ... other fields\n    Plugins      Plugins\n    PluginConfig []PluginConfig\n}\n\ntype Plugins struct {\n    QueueSort      []Plugin\n    PreFilter      []Plugin\n    Filter         []Plugin\n    PreScore       []Plugin\n    Score          []Plugin\n    Reserve        []Plugin\n    Permit         []Plugin\n    PreBind        []Plugin\n    Bind           []Plugin\n    PostBind       []Plugin\n    UnReserve      []Plugin\n}\n\ntype Plugin struct {\n    Name   string\n    Weight int // Only valid for Score plugins\n}\n\ntype PluginConfig struct {\n    Name string\n    Args runtime.Unknown\n}\n```\n\nExample:\n\n```json\n{\n  \"plugins\": {\n    \"preFilter\": [\n      {\n        \"name\": \"PluginA\"\n      },\n      {\n        \"name\": \"PluginB\"\n      },\n      {\n        \"name\": \"PluginC\"\n      }\n    ],\n    \"score\": [\n      {\n        \"name\": \"PluginA\",\n        \"weight\": 30\n      },\n      {\n        \"name\": \"PluginX\",\n        \"weight\": 50\n      },\n      {\n        \"name\": \"PluginY\",\n        \"weight\": 10\n      }\n    ]\n  },\n  \"pluginConfig\": [\n    {\n      \"name\": \"PluginX\",\n      \"args\": {\n        \"favorite_color\": \"#326CE5\",\n        \"favorite_number\": 7,\n        \"thanks_to\": \"thockin\"\n      }\n    }\n  ]\n}\n```\n\n### Enable/Disable\n\nWhen specified, the list of plugins for a particular extension point are the\nonly ones enabled. If an extension point is omitted from the config, then the\ndefault set of plugins is used for that extension point.\n\n### Change Evaluation Order\n\nWhen relevant, plugin evaluation order is specified by the order the plugins\nappear in the configuration. A plugin that registers for multiple extension\npoints can have different ordering at each extension point.\n\n### Optional Args\n\nPlugins may receive arguments from their config with arbitrary structure.\nBecause one plugin may appear in multiple extension points, the config is in a\nseparate list of `PluginConfig`.\n\nFor example,\n\n```json\n{\n   \"name\": \"ServiceAffinity\",\n   \"args\": {\n      \"LabelName\": \"app\",\n      \"LabelValue\": \"mysql\"\n   }\n}\n```\n\n```go\nfunc NewServiceAffinity(args *runtime.Unknown, h FrameworkHandle) (Plugin, error) {\n    if args == nil {\n        return nil, errors.Errorf(\"cannot find service affinity plugin config\")\n    }\n    if args.ContentType != \"application/json\" {\n        return nil, errors.Errorf(\"cannot parse content type: %v\", args.ContentType)\n    }\n    var config struct {\n        LabelName, LabelValue string\n    }\n    if err := json.Unmarshal(args.Raw, \u0026config); err != nil {\n        return nil, errors.Wrap(err, \"could not parse args\")\n    }\n    //...\n}\n```\n\n### Backward compatibility\n\nThe current `KubeSchedulerConfiguration` kind has `apiVersion:\nkubescheduler.config.k8s.io/v1alpha1`. This new config format will be either\n`v1alpha2` or `v1beta1`. When a newer version of the scheduler parses a\n`v1alpha1`, the \"policy\" section will be used to construct an equivalent plugin\nconfiguration.\n\n*Note: Moving `KubeSchedulerConfiguration` to `v1` is outside the scope of this\ndesign, but see also\nhttps://github.com/kubernetes/enhancements/blob/master/keps/sig-cluster-lifecycle/wgs/0032-create-a-k8s-io-component-repo.md\nand https://github.com/kubernetes/community/pull/3008*\n\n## Interactions with Cluster Autoscaler\n\nThe Cluster Autoscaler will have to be changed to run Filter plugins instead of predicates. \nThis can be done by creating a Framework instance and invoke `RunFilterPlugins`.\n\n# USE CASES\n\nThese are just a few examples of how the scheduling framework can be used.\n\n## Coscheduling\n\nFunctionality similar to\n[kube-batch](https://github.com/kubernetes-sigs/kube-batch) (sometimes called\n\"gang scheduling\") could be implemented as a plugin. For pods in a batch, the\nplugin would \"accumulate\" pods in the [permit](#permit) phase by using the\n\"wait\" option. Because the permit stage happens after [reserve](#reserve),\nsubsequent pods will be scheduled as if the waiting pod is using those\nresources. Once enough pods from the batch are waiting, they can all be\napproved.\n\n## Dynamic Resource Binding\n\n[Topology-Aware Volume Provisioning](https://kubernetes.io/blog/2018/10/11/topology-aware-volume-provisioning-in-kubernetes/)\ncan be (re)implemented as a plugin that registers for [filter](#filter) and\n[pre-bind](#pre-bind) extension points. At the filtering phase, the plugin can\nensure that the pod will be scheduled in a zone which is capable of provisioning\nthe desired volume. Then at the pre-bind phase, the plugin can provision the\nvolume before letting scheduler bind the pod.\n\n## Custom Scheduler Plugins (out of tree)\n\nThe scheduling framework allows people to write custom, performant scheduler\nfeatures without forking the scheduler's code. To accomplish this, developers\njust need to write their own `main()` wrapper around the scheduler. Because\nplugins must be compiled with the scheduler, writing a wrapper around `main()`\nis necessary in order to avoid modifying code in `vendor/k8s.io/kubernetes`.\n\n```go\nimport (\n    scheduler \"k8s.io/kubernetes/cmd/kube-scheduler/app\"\n)\n\nfunc main() {\n    command := scheduler.NewSchedulerCommand(\n            scheduler.WithPlugin(\"example-plugin1\", ExamplePlugin1),\n            scheduler.WithPlugin(\"example-plugin2\", ExamplePlugin2))\n    if err := command.Execute(); err != nil {\n        fmt.Fprintf(os.Stderr, \"%v\\n\", err)\n        os.Exit(1)\n    }\n}\n```\n\n*Note: The above code is an example, and might not match the latest implemented API.*\n\nThe custom plugins would be enabled as normal plugins in the scheduler config, see [Configuring Plugins](#configuring-plugins).\n\n# TEST PLANS\n\nThe scheduling framework is expected to be backward compatible with the existing\nKubernetes scheduler. As a result, we expect all the existing tests of the\nscheduler to pass during and after the framework is developed.\n\n* Unit Tests\n  * Each plugin developed for the framework is expected to have its own unit\ntests with reasonable coverage.\n\n* Integration Tests\n  * As we build extension points, we must add appropriate integration tests that\nensure plugins registered at these extension points are invoked and\nthe framework processes their return values correctly.\n  * If a plugin adds a new functionality that didn't exist in the past, it must be\naccompanied by integration tests with reasonable coverage.\n\n* End-to-end tests\n  * End-to-end tests should be added for new scheduling features and plugins that\ninteract with external components of Kubernetes. For example, if a plugin needs\nto interact with the API server and Kubelets, end-to-end tests may be needed.\nEnd-to-end tests are not needed when integration tests can provided adequate coverage.  \n\n# GRADUATION CRITERIA\n\n* Alpha\n  * Extension points for `Reserve`, `Unreserve`, and `Prebind` are built.\n  * Integration tests for these extension points are added.\n\n* Beta\n  * All the extension points listed in this KEP and their corresponding tests\n  are added.\n  * Persistent dynamic volume binding logic is converted to a plugin.\n\n* Stable\n  * Existing 'Predicate' and 'Priority' functions and preemption logic are\n  converted to plugins.\n  * No major bug in the implementation of the framework is reported in the past\n  three months.\n\n# IMPLEMENTATION HISTORY\n\nTODO: write down milestones and target releases, and a plan for how we will\ngracefully move to the new system\n"
+  },
+  {
+    "id": "a0bf3c49f6e125d4a9daea3bbf0be967",
+    "title": "Promote Pod Priority and Preemption to GA",
+    "authors": ["@bsalamat"],
+    "owningSig": "sig-scheduling",
+    "participatingSigs": ["sig-scheduling"],
+    "reviewers": ["@k82cn"],
+    "approvers": ["@liggitt"],
+    "editor": "Babak Salamat",
+    "creationDate": "2019-01-31",
+    "lastUpdated": "2019-01-31",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Promote Pod Priority and Preemption to GA\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Testing Plan](#testing-plan)\n  - [Unit Tests](#unit-tests)\n  - [Integration tests](#integration-tests)\n  - [E2E tests](#e2e-tests)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nPod Priority and Preemption are features introduced in Kubernetes 1.8 as alpha features and \npromoted to beta in 1.11. Pod Priority enables users to specify importance of\na Pod. Pods with higher priority are scheduled ahead of other pods with\nlower priority. When a cluster does not have enough capacity for running a high\npriority pod, the scheduler preempts and removes lower priority pods in order to\nmake room for the high priority pod.\n\n## Motivation\n\nPod Priority and Preemption have existed in the past several releases and some\nof our most critical components of Kubernetes, i.e. critical DaemonSet Pods,\nrely on this feature for guaranteed scheduling since Kubernetes 1.12.\n\n### Goals\n\nPromote Pod Priority and Preemption to GA.\n\n### Non-Goals\n\nMake any change of functionality to the features.\n\n## Proposal\n\nCreate `scheduling.k8s.io/v1` API group and add `PriorityClass` to it.\n\nMake necessary changes to our code base to use `scheduling.k8s.io/v1` instead of\n`scheduling.k8s.io/v1beta1`.\n\nUpdate our documentation to reflect the new version and status of the features.\n\n### Risks and Mitigations\n\nGiven that there is no functionality changes, we don't expect any logical errors\ncaused by this change.\n\n## Graduation Criteria\n\n* The features have been stable and reliable in the past several releases.\n* Adequate documentation exists for the features.\n* Test coverage of the features is acceptable.\n\n## Testing Plan\nPod priority and preemption have unit, integration, and e2e tests. These tests\nare run regularly as a part of Kubernetes presubmit and CI/CD pipeline.\n\n### Unit Tests\nHere is a list of unit tests for various modules of the feature:\n* [Priority admission controller tests](https://github.com/kubernetes/kubernetes/blob/master/plugin/pkg/admission/priority/admission_test.go)\n* [Priority aware scheduling queue tests](https://github.com/kubernetes/kubernetes/blob/master/pkg/scheduler/internal/queue/scheduling_queue_test.go)\n* [Scheduler preemption tests](https://github.com/kubernetes/kubernetes/blob/master/pkg/scheduler/core/generic_scheduler_test.go).\nThis file includes other tests too.\n\n### Integration tests\nIntegration tests for priority and preemption are [found here](https://github.com/kubernetes/kubernetes/blob/master/test/integration/scheduler/preemption_test.go).\n\n### E2E tests\nEnd to end tests for priority and preemption are [found here](https://github.com/kubernetes/kubernetes/blob/master/test/e2e/scheduling/preemption.go).\n\n## Implementation History\n\nPod Priority and Preemption are tracked as part of [enhancement#564](https://github.com/kubernetes/enhancements/issues/564).\nThe proposal for Pod Priority can be [found here](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/scheduling/pod-priority-api.md)\nand Preemption proposal is [here](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/scheduling/pod-preemption.md).\n"
+  },
+  {
+    "id": "2c62a05f5a4e0fb6195dc6509bdac6d6",
+    "title": "Even Pods Spreading",
+    "authors": ["@Huang-Wei"],
+    "owningSig": "sig-scheduling",
+    "participatingSigs": null,
+    "reviewers": ["@bsalamat", "@lavalamp", "@krmayankk", "@ahg-g", "@alculquicondor"],
+    "approvers": ["@ahg-g", "@alculquicondor"],
+    "editor": "",
+    "creationDate": "2019-02-21",
+    "lastUpdated": "2020-01-21",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Pod Topology Spread\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Terms](#terms)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Story 1](#story-1)\n    - [Story 2](#story-2)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [API](#api)\n    - [Option 1](#option-1)\n    - [Option 2 (preferred)](#option-2-preferred)\n  - [MaxSkew](#maxskew)\n  - [How User Stories are Addressed](#how-user-stories-are-addressed)\n  - [Pros/Cons](#proscons)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n- [Alternatives](#alternatives)\n- [Impact to Other Features](#impact-to-other-features)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [x] KEP approvers have set the KEP status to `implementable`\n- [x] Design details are appropriately documented\n- [x] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [x] Graduation criteria is in place\n- [x] \"Implementation History\" section is up-to-date for milestone\n- [x] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Terms\n\n- **Topology:** describe a series of worker nodes which belongs to the same\n  region/zone/rack/hostname/etc. In terms of Kubernetes, they're defined and\n  grouped by node labels.\n- **Affinity**: if not specified particularly, \"Affinity\" refers to\n  `NodeAffinity`, `PodAffinity` and `PodAntiAffinity`.\n- **CA**: Cluster Autoscaler.\n  [CA](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler)\n  is a tool that automatically adjusts the size of the Kubernetes cluster upon\n  specific conditions.\n\n## Summary\n\nThe `PodTopologySpread` feature gives users more fine-grained control on\ndistribution of pods scheduling, so as to achieve better high availability and\nresource utilization.\n\n## Motivation\n\nIn Kubernetes, \"Affinity\" related directives are aimed to control how pods are\nscheduled - more packed or more scattering. But right now only limited options\nare offered: for `PodAffinity`, infinite pods can be stacked onto qualifying\ntopology domain(s); for `PodAntiAffinity`, only one pod can be scheduled onto a\nsingle topology domain.\n\nThis is not an ideal situation if users want to put pods evenly across different\ntopology domains - for the sake of high availability or saving cost. And regular\nrolling upgrade or scaling out replicas can also be problematic. See more\ndetails in [user stories](#user-stories).\n\n### Goals\n\n- Pod Topology Spread Constraints is calculated among pods instead of apps API (such as\n  Deployment, ReplicaSet).\n- Pod Topology Spread Constraints can be either a predicate (hard requirement) or a priority\n  (soft requirement).\n\n### Non-Goals\n\n- Pod Topology Spread Constraints is NOT calculated on an application basis. In other words, it's\n  not only applied within replicas of an application, but also applied to\n  replicas of other applications if appropriate.\n- \"Max number of pods per topology\" is NOT a goal.\n- Scale-down on an application is not guaranteed to achieve desired pods spreading\n  in the initial implementation.\n\n## Proposal\n\n### User Stories\n\n#### Story 1\n\nAs an application developer, I want my application pods to be scheduled onto\nspecific topology domains as even as possible. Current status is that pods may\nbe stacked onto a specific topology domain. (see\n[#68981](https://github.com/kubernetes/kubernetes/issues/68981))\n\n#### Story 2\n\nAs an application developer, I want my application pods not to co-exist with\nspecific pods (via PodAntiAffinity). But in some cases, it'd be favorable to\ntolerate \"violating\" pods in a manageable way. For example, suppose an app\n(replicas=2) is using PodAntiAffinity and deployed onto a 2-nodes cluster, and\nnext the app needs to perform a rolling upgrade, then a third replacement pod is\ncreated, but it failed to be placed due to lack of resource. In this case,\n\n- if CA is enabled, a new machine will be provisioned to hold the new pod\n  (although old replicas will be deleted afterwards) (see\n  [#40358](https://github.com/kubernetes/kubernetes/issues/40358))\n- if CA is not enabled, it's a deadlock since the replacement pod can't be\n  placed. The only workaround at this moment is to update app strategyType from\n  \"RollingUpdate\" to \"Recreate\".\n\nNeither of them is an ideal solution. A promising solution is to give user an\noption to trigger \"toleration\" mode when the cluster is out of resource. Then in\naforementioned example, a third pod is \"tolerated\" to be put onto node1 (or\nnode2). But keep it in mind, this behavior is only triggered upon resource\nshortage. For a 3-nodes cluster, the third pod will still be placed onto node3\n(if node3 is capable).\n\n### Risks and Mitigations\n\nThe feature requires additional processing for pods that use it and it is ok to\nhave some performance overhead. But we will make sure our implementation will\nnot have any performance penalty for pods that do not use this feature.\n\n## Design Details\n\n### API\n\nA new structure called `TopologySpreadConstraint` is introduced which acts as a\nstandalone spec and is applied to `pod.spec`. It's only effective when it's not\nnil.\n\n```go\ntype PodSpec struct {\n    ......\n    // TopologySpreadConstraints describes how a group of pods are spread\n    // If specified, scheduler will enforce the constraints\n    // +optional\n    TopologySpreadConstraints []TopologySpreadConstraint\n    ......\n}\n```\n\n#### Option 1\n\nInside `TopologySpreadConstraint`, we need hard affinityTerms (similar with\n`PodAffinityTerm`) and soft affinityTerms (similar with\n`WeightedPodAffinityTerm`). This describes when we perform even distribution,\nwhich pods are considered as a group.\n\n```go\ntype TopologySpreadConstraint struct {\n    // MaxSkew describes the degree of imbalance of pods spreading.\n    // It's the max difference between the number of matching pods in any two\n    // topology domains of a given topology type.\n    // Default value is 1 and 0 is not allowed.\n    MaxSkew int32\n    // TopologyKey defines where pods are placed evenly\n    TopologyKey string\n    // Similar with the same field in PodAffinity/PodAntiAffinity\n    // +optional\n    RequiredDuringSchedulingIgnoredDuringExecution []PodAffinityTerm\n    // Similar with the same field in PodAffinity/PodAntiAffinity\n    // +optional\n    PreferredDuringSchedulingIgnoredDuringExecution []WeightedPodAffinityTerm\n}\n```\n\n#### Option 2 (preferred)\n\nAnother option is to flatten \"required\" and \"preferred\" podAffinityTerms, and\neliminate embedded \"TopologyKey\":\n\n```go\ntype UnsatisfiableConstraintResponse string\n\nconst (\n    // do not schedule a pod in all circumstances\n    DoNotSchedule UnsatisfiableConstraintResponse = \"DoNotSchedule\"\n    // schedule a pod despite of any circumstance\n    ScheduleAnyway UnsatisfiableConstraintResponse = \"ScheduleAnyway\"\n)\n\ntype TopologySpreadConstraint struct {\n    // MaxSkew describes the degree of imbalance of pods spreading.\n    // It's the max difference between the number of matching pods in any two\n    // topology domains of a given topology type.\n    // For example, in a 3-zone cluster, currently pods with the same labelSelector\n    // are spread as 1/1/0:\n    // - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 1/1/1;\n    // schedule it onto zone1(zone2) will make the ActualSkew(2) violates MaxSkew(1)\n    // - if MaxSkew is 2, incoming pod can be scheduled to any zone.\n    // Default value is 1 and 0 is not allowed.\n    MaxSkew int32\n    // TopologyKey is the key such that we consider each value as a \"bucket\";\n    // we try to put balanced number of pods into each bucket.\n    TopologyKey string\n    // WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy\n    // the spreading constraint.\n    // - DoNotSchedule (default) tells the scheduler not to schedule it\n    // - ScheduleAnyway tells the scheduler to still schedule it\n    // Note: it's considered as \"Unsatisfiable\" only when actual skew on all nodes\n    // exceeds \"MaxSkew\".\n    WhenUnsatisfiable UnsatisfiableConstraintResponse\n    // Label selector for pods. This's enforced by scheduler to check which pods\n    // should be recognized as a group to satisfy the spreading constraint.\n    Selector *metav1.LabelSelector\n}\n```\n\n### MaxSkew\n\n`MaxSkew` is the core of this KEP, so the exact semantics are clarified as below:\n\n- how Skew is calculated and enforced\n\nSuppose we have a 3-zone cluster, currently pods with the same labelSelector are\nspread as 1/1/0. Internally we compute an \"ActualSkew\" for each topology\ndomain representing \"matching pods in this topology domain\" minus \"minimum\nmatching pods in any topology domain\", so for this 1/1/0 cluster, the ActualSkew\nfor each zone is 1(1-0)/1(1-0)/0(0-0). (If the spreading is 3/2/1, the\nActualSkew for each zone will be 2(3-1)/1(2-1)/0(1-1))\n\nThe internal computation logic would be to find nodes satisfying \"ActualSkew \u003c=\nMaxSkew\". Let's go back to the 1/1/0 example:\n\nIf MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 1/1/1;\nbecause schedule it onto zone1(zone2) will make the ActualSkew(2) violates\nMaxSkew(1).\n\nIf MaxSkew is 2, incoming pod can be scheduled to any zone.\n\n**NOTE:** If NodeAffinity or NodeSelector is defined, spreading is applied to\nnodes that pass those filters. For example, if NodeAffinity chooses zone1 and\nzone2 and there are 10 zones in the cluster, pods are spread in zone1 and zone2\nonly and MaxSkew is enforced only on these two zones.\n\n- chicken/egg problem\n\nLet's say we have a 3-zone cluster, and there is no pod in any node yet. Here\ncomes a pod, and it wants to be scheduled to a zone which has pods with label\n`foo`. Obviously, there is no qualified node. However, we don't stop here;\ninstead, we proceed to check if the incoming pod matches itself on its labels.\nIf it does, we would think any node is a fit.\n\nThis is actually an existing implication in PodAffinity algorithm. I just want\nto put here again to avoid confusion. And **below examples are all based the\nassumption that incoming pod matches itself on its labels**.\n\n- matching number and min matching number\n\n\"matching\" number is the number of pods matched on topology domain (defined by\nthe global topologyKey). Suppose we have a 3-zone cluster, and there are 3 pods\nin zone1, 2 pods in zone2, 1 pod in zone3. And all pods carry label `foo`:\n\n```\n|            zone1           |            zone2           |  zone3 |\n| node1a |  node1b  | node1c |  node2a  | node2b | node2c | node3a |\n|   pod  | pod, pod |        | pod, pod |        |        |   pod  |\n```\n\nNow let's say there comes a pod, it wants to be placed along with pods which\ncarries label `foo` in zones.\n\nIf global topologyKey is \"zone\" and maxSkew is \"1\", then incoming pod can only\nbe put into zone3 because for zone1, it violates `matching num (3) - min\nmatching num (1) \u003c maxSkew (1)`. Zone2 violate the formula the same way.\n\nIf global topologyKey is \"node\" and maxSkew is \"1\", things are slightly\ndifferent. Min matching num becomes 0 now, and hence only node1c, node2b and\nnode2c are qualified candidates.\n\n- what if a topology domain is infeasible\n\nSuppose we have pods distribution in a 3-zone cluster as 3/3/0, and all pods\nhave label `foo`:\n\n```\n|    zone1    |    zone2    | zone3 (infeasible) |\n| pod,pod,pod | pod,pod,pod |                    |\n```\n\nAnd we have an incoming pod which wants to be scheduled with pods which carry\nlabel `foo` in zones. And suppose all nodes in zone3 are infeasible, e.g. due to\ntaints or lack of resources. In this case:\n\nIf it's a hard requirement, we treat the `min matching num` as 0, which means\nincoming pod would fail to be scheduled.\n\nIf it's a soft requirement, we treat the `min matching num` as 3 instead of 0,\nwhich means incoming pod can be placed onto zone1 or zone2.\n\n- (more cases) when a topology domain is infeasible\n\n    \u003e Suppose maxSkew is 1: (~~zone~~ means the zone is infeasible)\n\n    - for a \"1/1/~~0~~\" cluster, pod can't be placed onto any zone if it's a\n      Predicate; zone1 and zone2 are equally preferred if it's a Priority\n    - for a \"2/1/~~0~~\" cluster, pod can't be placed onto any zone if it's a\n      Predicate; zone2 is preferable over zone1 if it's a Priority\n    - for a \"1/1/~~1~~\" cluster, pod can be placed onto zone1 or zone2 if it's a\n      Predicate; zone1 and zone2 are equally preferred if it's a Priority\n    - for a \"2/1/~~1~~\" cluster, pod can be placed onto zone2 if it's a\n      Predicate; zone2 is preferable over zone1 if it's a Priority\n\n- when formula check is enforced\n\nWe only enforce the formula check upon new pod scheduling. In other words, if\npods become imbalanced (due to explicit taints, lack of resources, or node\nlost), we don't do proactive re-scheduling. Our goal is to not make things\nworse.\n\n### How User Stories are Addressed\n\nIn terms of story 1, users can define a `TopologySpreadConstraint` to achieve an\neven pods distribution:\n\n```yaml\nspec:\n  topologySpreadConstraint:\n    maxSkew: 1\n    topologyKey: k8s.io/zone\n    whenUnsatisfiable: DoNotSchedule\n    selector:\n      matchLabels:\n        app: foo\n```\n\nAnd it can work together with NodeSelector/NodeAffinity. (check\n[MaxSkew](#maxskew) for more details)\n\nSimilarly, story 2 can also be addressed using above solution.\n\nAnd the pseudo algorithms below explain the processing flow in a nutshell.\n\n- Predicate\n\n```bash\nfor each candidate node; do\n    if \"TopologySpreadConstraint\" is enabled for the pod being scheduled; then\n        # minMatching num is globally calculated\n        count number of matching pods on the topology domain this node belongs to\n        if \"matching num - minMatching num\" \u003c \"MaxSkew\"; then\n            approve it\n        fi\n    fi\ndone\n```\n\n- Priority\n\n```bash\nfor each candidate node; do\n    if \"TopologySpreadConstraint\" is enabled for the pod being scheduled; then\n        # minMatching num is calculated across node list filtered by Predicate phase\n        count number of matching pods on the topology domain this node belongs to\n        calculate the value of \"matching num - minMatching num\" minus \"MaxSkew\"\n        the lower, the higher score this node is ranked\n    fi\ndone\n```\n\n### Pros/Cons\n\n**Pros:**\n\n- Independent design, so can work independently with Affinity API\n- Support both predicate and priority\n\n**Cons:**\n\n- Work for Story 2 without the presence of PodAntiAffinity\n- More API changes\n- More code changes, and some efforts of refactoring code to ensure Affinity\n  related structure/logic can be reused gracefully\n\n### Test Plan\n\nTo ensure this feature to be rolled out in high quality. Following tests are mandatory:\n\n- **Unit Tests:** All core changes must be covered by unit tests.\n- **Integration Tests / E2E Tests:** All user cases discussed in this KEP must\n  be covered by either integration tests or e2e tests.\n- **Benchmark Tests:** We can bear with slight performance overhead if users are\n  using this feature, but it shouldn't impose penalty to users who are not using\n  this feature. We will verify it by designing some benchmark tests.\n\n### Graduation Criteria\n\nAlpha:\n\n- [x] This feature will be rolled out as an Alpha feature in v1.15.\n- [x] API changes and feature gating.\n- [x] Necessary defaulting, validation and generated code.\n- [x] Predicate implementation.\n- [x] Priority implementation.\n- [x] Implementation of all scenarios discussed in this KEP.\n- [x] Minimum viable test cases mentioned in [Test Plan](#test-plan) section.\n\nBeta:\n\n- [ ] This feature will be enabled by default as a Beta feature in v1.18.\n- [ ] Replace of the term \"Even Pods Spreading\" with \"Pod Topology Spread\n  Constraints\" in docs, KEP and source code. However, keep the feature gate name\n  \"EvenPodsSpread\" as is.\n- [ ] Migrate predicate implementation to preFilter / filter plugins.\n- [ ] Migrate priority implementation to postFilter / score plugins.\n- [ ] Calculate \"preFilterState\" if it's not pre-calculated in preFilter plugin.\n  This is particularly for some extended usage such as Cluster Autoscaler.\n- [ ] Add necessary end-to-end tests.\n\n## Alternatives\n\n- mixin new fields into `pod.spec.affinity` to act as a\"sub feature\" of Affinity\n\n    ```go\n    type TopologySpreadConstraint struct {\n        // MaxSkew describes the degree of imbalance of pods spreading.\n        // Default value is 1 and 0 is not allowed.\n        MaxSkew int32\n        // TopologyKey defines where pods are placed evenly\n        TopologyKey string\n    }\n\n    type NodeAffinity struct {\n        TopologySpreadConstraint *TopologySpreadConstraint\n        ......\n    }\n\n    type PodAffinity struct {\n        TopologySpreadConstraint *TopologySpreadConstraint\n        ......\n    }\n\n    type PodAntiAffinity struct {\n        TopologySpreadConstraint *TopologySpreadConstraint\n        ......\n    }\n    ```\n\n    - Pros:\n        - Less API changes\n        - Less code changes (code can be built on existing InterPodPredicate, as\n          well as the internal data structures)\n    - Cons:\n        - The support on NodeAffinity is vague\n        - Current API design only supports predicate\n\n## Impact to Other Features\n\nThe motivation of this KEP is to resolve limitations of existing features, but\nit won't replace them.\n\nComparing to this feature, PodAffinity has the most expressive APIs such like\nmultiple podAffinityTerms and multiple topologyKeys, hence still fits for the\ncomplex scenarios; PodAntiAffinity still fits for the scenario which needs to\nplace up to one pod to one topology domain.\n\nHowever there are some notices worth mentioning for efficient cooperation with\nexisting features.\n\n- NodeAffinity/NodeSelector\n\nAs aforementioned, it's a reasonable assumption that evenness should be applied\namong the filtered nodes specified by NodeAffinity/NodeSelector. So be aware of\nimplicit assumption.\n\n- PodAffinity\n\nPodAffinity can work seamlessly with this feature. But a tip here is that if\nyour requirement on PodAffinity only applies to one topology, and cares about\nevenness, you can simply put the podAffinityTerm in the manner of `selector` and\n`topologyKey` of `TopologySpreadConstraint`. This can achieve the same\nscheduling goal efficiently.\n\n- PodAntiAffinity\n\n(not specific to this KEP, but worth mentioning here)\n\nCurrently PodAntiAffinity supports arbitrary topology domain, but sadly this\ncauses a slow down in scheduling (see [Rethink pod\naffinity/anti-affinity](https://github.com/kubernetes/kubernetes/issues/72479)).\nWe're evaluating solutions such as limit topology domain to node, or internally\nimplement a fast/slow path handling that. If this KEP gets implemented, we can\nsimply achieve the semantics of \"PodAntiAffinity in zones\" via a combination of\n\"Even pods spreading in zones\" plus \"PodAntiAffinity in nodes\" which could be an\nextra benefit of this KEP.\n\n## Implementation History\n\n- 2019-02-21: Initial KEP sent out for review.\n- 2019-04-16: Initial KEP approved.\n- 2019-05-01: First [KEP implementation PR](https://github.com/kubernetes/kubernetes/pull/77327) sent out for review.\n- 2020-01-21: KEP updated to meet the criteria of promoting to beta.\n  - NOTE: The term \"Even Pods Spreading\" is replaced with \"Pod Topology Spread\",\n    to be consistent with the [official doc](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/), but\n    the featuregate name \"EvenPodsSpread\" remains unchanged.\n"
+  },
+  {
+    "id": "5bbbea9f5984883719697297ab03caa9",
+    "title": "Extending RequestedToCapacityRatio Priority Function to support Resource Bin Packing of Extended Resources - @sudeshsh",
+    "authors": null,
+    "owningSig": "sig-scheduling",
+    "participatingSigs": ["sig-scheduling"],
+    "reviewers": ["@k82cn", "@Huang-Wei", "@bsalamat"],
+    "approvers": ["@k82cn", "@bsalamat"],
+    "editor": "",
+    "creationDate": "2019-03-11",
+    "lastUpdated": "2019-05-01",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# ExtendingRequestedToCapacityRatio Priority Function to support Resource Bin Packing of Extended Resources\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Argument Input Scenarios](#argument-input-scenarios)\n  - [Design Details](#design-details)\n  - [User Stories](#user-stories)\n    - [Story 1](#story-1)\n    - [Default Behavior](#default-behavior)\n    - [Extender Resource Scheduler Behavior](#extender-resource-scheduler-behavior)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n    - [Examples](#examples)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nExtend RequestedToCapacityRatio Priority Function to allow users to use the best fit polices during scheduling. It will allow users to apply [bin packing](https://en.wikipedia.org/wiki/Bin_packing_problem) on core resources like CPU, Memory as well as extended resources like accelerators.\n\n## Motivation\n\nWhile running workloads on kubernetes which use accelerator devices the default scheduler spreads pods across nodes resulting in fragmentation of extended resources. This fragmentation prevents scheduling pods with larger device resource requirements, and they remain in the pending state.\n\n### Goals\n\n- Schedule Pods using BestFit Policy using RequestedToCapacityRatio Priority Function for extended Resources\n- Reduce Fragmentation of scarce resources on the Cluster\n\n### Non-Goals\n\n-\n\n## Proposal\n\nThe plan is to modify  `requested_to_capacity_ratio` priority function to support `resource_bin_packing` of extended resources. Add another argument `resources` of type `map[v1.ResourceName]int64{}` .This would allow users who want to bin pack a resource to use the function by setting the argument resources which would require them to specify weights for bin packing. For example\n\n```yaml\n\"priorities\": [\n   ...{\n      \"name\":\"RequestedToCapacityRatioPriority\",\n      \"weight\":5,\n      \"argument\":{\n         \"requestedToCapacityRatioArguments\":{\n            \"shape\":[\n               {\n                  \"utilization\":100,\n                  \"score\":10\n               },\n               {\n                  \"utilization\":0,\n                  \"score\":0\n               }\n            ],\n            \"resources\":[\n               {\n                  \"resource\":\"intel.com/foo\",\n                  \"weight\":5\n               },\n               {\n                  \"resource\":\"intel.com/bar\",\n                  \"weight\":2\n               },\n               {\n                  \"resource\":\"cpu\",\n                  \"weight\":1\n               }\n            ]\n         }\n      }\n   }\n]\n```\n\nThe argument `resources` is optional and by default will be set to the value\n\n```go\nresources := map[v1.ResourceName]int64{\n  v1.ResourceMemory : 1,\n  v1.ResourceCPU    : 1,\n}\n```\n\n### Argument Input Scenarios\n\n1. If the resource does not exist the score for that resources defaults to 0\n2. As we are storing resources as a map there can be a single weight for every resource type.\n3. Negative weights are not allowed, this would be filtered inside the function and an appropriate error would be thrown.\n\n\n### Design Details\n\nThe node score would be calculated as (podRequest + requested) / allocatable. The weights would be used to calculate the resulting node score in the following way.\n\n```go\nresources := make(map[v1.ResourceName]weight)\nnodeScore := 0\nweightSum := 0\nfor resource, weight := range resources {\n  nodeScore += resourceScoringFunction((podRequest[resource]+ requested[resource]), allocatable[resource]) * weight\n  weightSum += weight\n}\nnodeScore = nodeScore / weightSum\n```\n\nUpdate the  function `buildRequestedToCapacityRatioScorerFunction` definition as\n\n```go\nfunc buildRequestedToCapacityRatioScorerFunction(scoringFunctionShape FunctionShape, resources []Resources) func(nodeInfo *schedulercache.NodeInfo) int64\n```\n\n\n### User Stories\n\n#### Story 1\n\nLet's consider a cluster with `intel.com/foo` as a scarce resource. A user needs to submit 3 `jobs` with specs as shown below\n\n![Test Scenario](20190311-resource_bin_packing_priority_function_scenario.png)\n\n#### Default Behavior\n\nThe default scheduler in most cases will schedule the Pods as follows as there is no priority function for an extended resource for bin packing.\n\n![Default Behavior](20190311-resource_bin_packing_priority_function_default.png)\n\n#### Extender Resource Scheduler Behavior\n\nThe Scheduler should submit the 2 resource job on Node 3 as the utilization is higher. This would reduce the fragmentation of extended resource and reduce pods in the pending state.\n\n![Extended Scheduler Behavior](20190311-resource_bin_packing_priority_function_extended.png)\n\n### Test Plan\n\n- **Unit Tests:** All changes must be covered by unit tests.\n- **Integration Tests:** The use cases discussed in this KEP must\n  be covered by integration tests.\n\n### Graduation Criteria\n\nAlpha:\n\n- [ ] This feature will be rolled out as an Alpha feature in v1.15\n- [ ] Necessary defaulting, validation\n- [ ] Adequate documentation for the changes\n- [ ] Minimum viable test cases mentioned in [Test Plan](#test-plan) section\n\n#### Examples\n\n```\nRequested Resources\n\nintel.com/foo : 2\nMemory: 256MB\nCPU: 2\n\nResource Weights\n\nintel.com/foo : 5\nMemory: 1\nCPU: 3\n\nFunctionShapePoint {{0, 0}, {100, 10}}\n\nNode 1 Spec\n\nAvailable:\nintel.com/foo : 4\nMemory : 1 GB\nCPU: 8\n\nUsed:\nintel.com/foo: 1\nMemory: 256MB\nCPU: 1\n\n\nNode Score:\n\nintel.com/foo  = resourceScoringFunction((2+1),4)\n               =  (100 - ((4-3)*100/4)\n               =  (100 - 25)\n               =  75\n               =  rawScoringFunction(75)\n               = 7\n\nMemory         = resourceScoringFunction((256+256),1024)\n               = (100 -((1024-512)*100/1024))\n               = 50\n               = rawScoringFunction(50)\n               = 5\n\nCPU            = resourceScoringFunction((2+1),8)\n               = (100 -((8-3)*100/8))\n               = 37.5\n               = rawScoringFunction(37.5)\n               = 4\n\nNodeScore   =  (7 * 5) + (5 * 1) + (4 * 3) / (5 + 1 + 3)\n            =  6\n\n\nNode 2 Spec\n\nAvailable:\nintel.com/foo: 8\nMemory: 1GB\nCPU: 8\n\nUsed:\n\nintel.com/foo: 2\nMemory: 512MB\nCPU: 6\n\n\nNode Score:\n\nintel.com/foo  = resourceScoringFunction((2+2),8)\n               =  (100 - ((8-4)*100/8)\n               =  (100 - 25)\n               =  50\n               =  rawScoringFunction(50)\n               = 5\n\nMemory         = resourceScoringFunction((256+512),1024)\n               = (100 -((1024-768)*100/1024))\n               = 75\n               = rawScoringFunction(75)\n               = 7\n\nCPU            = resourceScoringFunction((2+6),8)\n               = (100 -((8-8)*100/8))\n               = 100\n               = rawScoringFunction(100)\n               = 10\n\nNodeScore   =  (5 * 5) + (7 * 1) + (10 * 3) / (5 + 1 + 3)\n            =  7\n\n```\n\n## Implementation History\n\n- 2019-03-11: Initial KEP sent out for review.\n"
+  },
+  {
+    "id": "cb940839d50993c96a1ed125837c4253",
+    "title": "Pod affinity/anti-affinity supports Gt and Lt operators",
+    "authors": ["@wgliang"],
+    "owningSig": "sig-scheduling",
+    "participatingSigs": null,
+    "reviewers": ["@bsalamat", "@k82cn", "@Huang-Wei"],
+    "approvers": ["@bsalamat", "@k82cn"],
+    "editor": "",
+    "creationDate": "2019-02-22",
+    "lastUpdated": "2019-04-23",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n # Pod affinity/anti-affinity supports Gt and Lt operators\n\n ## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Correctness Tests](#correctness-tests)\n- [Integration Tests](#integration-tests)\n- [Performance Tests](#performance-tests)\n- [E2E tests](#e2e-tests)\n\u003c!-- /toc --\u003e\n\n#### Correctness Tests\nHere is a list of unit tests for various modules of the feature:\n\n- `NodeSelector` related tests\n- `PodSelector` related tests\n- `Gt` and `Lt` functional tests\n- `NumericAwareSelectorRequirement` related tests\n- Backwards compatibility - pods made with the new types should still be updatable if apiserver version is rolled back\n- Forwards compatibility - all pods created today are wire-compatible (both proto and json) with the new api\n\n#### Integration Tests\n- Integration tests for `PodSelector`\n\n#### Performance Tests\n- Performance test of `Gt` and `Lt` operators\n\n#### E2E tests\n- End to end tests for `PodSelector`\n\n ### Graduation Criteria\n\n _To be filled until targeted at a release._\n\n ## Implementation History\n\n - 2019-03-12: Initial KEP sent out for reviewing.\n"
+  },
+  {
+    "id": "a53351c514c7750d65810069febe44f1",
+    "title": "Add NonPreempting Option For PriorityClasses",
+    "authors": ["@vllry"],
+    "owningSig": "sig-scheduling",
+    "participatingSigs": ["sig-scheduling"],
+    "reviewers": ["k82cn", "wgliang"],
+    "approvers": ["bsalamat"],
+    "editor": "Vallery Lancey",
+    "creationDate": "2019-03-17",
+    "lastUpdated": "2019-03-28",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Allow PriorityClasses To Be Non-Preempting\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Testing Plan](#testing-plan)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\n[PriorityClasses](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/) are a GA feature as on 1.14,\nwhich impact the scheduling and eviction of pods.\nPods are be scheduled according to descending priority.\nIf a pod cannot be scheduled due to insufficient resources,\nlower-priority pods will be preempted to make room.\n\nThis proposal makes the preempting behavior optional for a PriorityClass,\nby adding a new field to PriorityClasses,\nwhich in turn populates PodSpec.\nIf a pod is waiting to be scheduled,\nand it does not have preemption enabled,\nit will not trigger preemption of other pods.\n\n## Motivation\n\nAllowing PriorityClasses to be non-preempting is important for running batch workloads.\n\nBatch workloads typically have a backlog of work,\nwith unscheduled pods.\nHigher-priority workloads can be assigned a higher priority via a PriorityClass,\nbut this may result in pods with partially-completed work being preempted.\nAdding the non-preempting option allows users to prioritize the scheduling queue,\nwithout discarding incomplete work.\n\n### Goals\n\nAdd a boolean to PriorityClasses,\nto enable or disable preemption for pods of that PriorityClass.\n\n### Non-Goals\n\n* Protecting pods from preemption. PodDisruptionBudget should be used.\n\n## Proposal\n\nAdd a Preempting field to both PodSpec and PriorityClass.\nThis field will default to true,\nfor backwards compatibility.\n\nIf Preempting is true for a pod,\nthe scheduler will preempt lower priority pods to schedule this pod,\nas is current behavior.\n\nIf Preempting is false,\na pod of that priority will not preempt other pods.\n\nSetting the Preempting field in PriorityClass provides a straightforward interface,\nand allows ResourceQuotas to restrict preemption.\n\nPriorityClass type example:\n```\ntype PriorityClass struct {\n\tmetav1.TypeMeta\n\tmetav1.ObjectMeta\n\tValue int32\n\tGlobalDefault bool\n\tDescription string\n\tPreempting *bool // New option\n}\n```\n\nThe Preempting field in PodSpec will be populated during pod admission,\nsimilarly to how the PriorityClass Value is populated.\nStoring the Preempting field in the pod spec has several benefits:\n* The scheduler does not need to be aware of PiorityClasses,\nas all relevant information is in the pod.\n* Mutating PriorityClass objects does not impact existing pods.\n* Kubelets can set Preempting on static pods.\n\nPodSpec type example:\n```\ntype PodSpec struct {\n    ...\n    Preempting *bool\n    ...\n}\n```\n\nThis feature should be gated in alpha, provisionally under the gate `NonPreemptingPriority`.\n\nDocumentation must be updated to reflect the new feature,\nand changes to PriorityClass/PodSpec fields.\n\n### Risks and Mitigations\n\nThe new feature may malfuction,\nor existing preemption functionality may be impaired.\nNew tests (covering both nonpreepting workloads and mixed workloads),\nand the existing preempting PriorityClass tests should be used to prove stability.\n\n## Graduation Criteria\n\n**Typical user story:**\nA user is running batch workloads on a cluster.\nThe user has a high-priority job,\nthat they wish to schedule before other workloads in the queue.\nAs the user does not want to preempt running batch workloads and discard work,\nthe user creates the new workload with a high-priority,\nnon-preempting PriorityClass.\nThe new workload's pods are scheduled ahead of the queue,\nwithout disrupting running workloads.\n\n* Users are able to run preempting and non-preempting workloads in a stable manner,\nand are not requesting additional changes.\n* The feature has been stable and reliable in at least 2 releases.\n* Adequate documentation exists for preemption and the optional field.\n* Test coverage includes non-preempting use cases.\n* Conformance requirements for non-preempting PriorityClasses are agreed upon.\n\n## Testing Plan\nAdd detailed unit and integration tests for nonpreempting workloads.\n\nAdd basic e2e tests, to ensure all components are working together.\n\nEnsure existing tests (for preempting PriorityClasses) do not break.\n\n## Implementation History\n\n[Original Github issue](https://github.com/kubernetes/kubernetes/issues/67671)\n\nPod Priority and Preemption are tracked as part of [enhancement#564](https://github.com/kubernetes/enhancements/issues/564).\nThe proposal for Pod Priority can be [found here](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/scheduling/pod-priority-api.md)\nand Preemption proposal is [here](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/scheduling/pod-preemption.md).\n"
+  },
+  {
+    "id": "cbcacd16dccfd353ffaed082c05d7c2a",
+    "title": "graduate-resourcequotascopeselectors-to-stable",
+    "authors": ["@ravisantoshgudimetla"],
+    "owningSig": "sig-api-machinery",
+    "participatingSigs": ["sig-scheduling", "sig-api-machinery"],
+    "reviewers": ["@bsalamat", "@k82cn", "@derekwaynecarr", "“@sjenning”", "@vikaschoudhary16"],
+    "approvers": ["@bsalamat", "@derekwaynecarr"],
+    "editor": "TBD",
+    "creationDate": "2019-04-23",
+    "lastUpdated": "2019-04-23",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Graduate ResourceQuotaScopeSelector to stable\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Notes](#implementation-notes)\n  - [Constraints](#constraints)\n  - [Test Plan](#test-plan)\n    - [Existing Tests](#existing-tests)\n    - [Needed Tests](#needed-tests)\n  - [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\n[ResourceQuotaScopeSelectors](https://kubernetes.io/docs/concepts/policy/resource-quotas/#resource-quota-per-priorityclass) has been created in the past to expand scopes to represent priorityClass names and their corresponding behaviour. This helps in removing the restriction of allowing critical pods to be created in `kube-system` namespace.\n\n## Motivation\n\n[Priority and Preemption](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/) has been GA'ed in 1.14 but with a caveat that critical pods could be created only in `kube-system` namespace. We wish to graduate `ResourceQuotaScopeSelectors` feature in order to overcome this limitation.\n\n### Goals\n\n* Plan to promote ResourceQuotaScopeSelectors to stable version.\n* Remove the limitation of creating critical pods only in `kube-system` namespace.\n\n### Non-Goals\n\n* Changing API field or meaning\n\n## Proposal\n\n### Implementation Notes\n\nIn the current implementation: \n\n1. Priority admission plugin is blocking [creation of critical pods](https://github.com/kubernetes/kubernetes/blob/90fbbee12950f336db2da94dda7beb87846f94e0/plugin/pkg/admission/priority/admission.go#L150) in namespaces other than `kube-system`.\n\n2. We don't have a default quota at bootstrap phase with scope selector to restrict critical pods to be created in `kube-system`.\n\nThe current implementation can be changed to relax the restriction of creating critical pod within `kube-system` namespace and let this restriction be created as a default quota at cluster bootstrap phase automatically.\n\nThis ensures:\n\n1. We are backwards compatible.\n2. System is not being abused where any regular user can create a critical pod in namespace of his/her own choice with those pods having capability to  displace control-plane or other critical pods.\n2. Cluster-admin can create quota in whatever the namespace he/she wants instead of limiting critical pods creation to `kube-system` namespace. The default quota with scope selectors is used in `kube-system` namespace.\n\n### Constraints\n\nWe should verify the automatic creation of quota and see if it causes any problems with quotas created in other namespaces. \n\n### Test Plan\n\n#### Existing Tests\n- [Run or Not](https://github.com/kubernetes/kubernetes/blob/90fbbee12950f336db2da94dda7beb87846f94e0/test/e2e/apimachinery/resource_quota.go#L799) tests the resource quota under different scenarios to check if the creation/deletion of resource quota with scope selectors is working or not.\n\n#### Needed Tests\n\n- Conformance tests need to be added for default quota with ResourceQuotaScopeSelectors.\n\n### Graduation Criteria\n\n- [ ] Remove limitation of critical pod creation in `kube-system` namespace in pod priority admission plugin\n- [ ] Create a `AdmissionConfiguration` object with `limitedResources` to prevent creation of system critical pods in all namespaces\n- [ ] Add a default quota with scope selector to allow critical pods to be created in `kube-system` namespace only\n- [ ] Graduate ResourceQuotaScopeSelectors API to GA\n- [ ] Needs a conformance test\n- [ ] Update documents to reflect the changes\n\n## Implementation History\n\n- ResourceQuotaScopeSelectors was introduced as alpha in kubernetes 1.11\n- In Kuberenetes 1.12 this feature was promoted to Beta\n"
+  },
+  {
+    "id": "f9d826525383d6ac638e0d2943064476",
+    "title": "Default Even Pod Spreading",
+    "authors": ["@alculquicondor"],
+    "owningSig": "sig-scheduling",
+    "participatingSigs": null,
+    "reviewers": ["@ahg-g", "@Huang-Wei"],
+    "approvers": ["@ahg-g", "@k82cn"],
+    "editor": "",
+    "creationDate": "2019-09-26",
+    "lastUpdated": "2010-10-07",
+    "status": "implementable",
+    "seeAlso": ["/keps/sig-aaa/20190221-even-pods-spreading.md"],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Default Even Pod Spreading\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Story 1](#story-1)\n    - [Story 2](#story-2)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n    - [Feature gate](#feature-gate)\n    - [Relationship with \u0026quot;DefaultPodTopologySpread\u0026quot; plugin](#relationship-with-defaultpodtopologyspread-plugin)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [API](#api)\n  - [Default constraints](#default-constraints)\n  - [How user stories are addressed](#how-user-stories-are-addressed)\n  - [Implementation Details](#implementation-details)\n    - [In the metadata/predicates/priorities flow](#in-the-metadatapredicatespriorities-flow)\n    - [In the scheduler framework](#in-the-scheduler-framework)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n    - [Alpha (v1.18):](#alpha-v118)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [x] KEP approvers have set the KEP status to `implementable`\n- [x] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nWith [Even Pods Spreading](/keps/sig-scheduling/20190221-even-pods-spreading.md),\nworkload authors can define spreading rules for their loads based on the topology of the clusters. \nThe spreading rules are defined in the `PodSpec`, thus they are tied to the pod.\n\nWe propose the introduction of configurable default spreading constraints, i.e. constraints that\ncan be defined at the cluster level and are applied to pods that don't explicitly define spreading constraints.\nThis way, all pods can be spread according to (likely better informed) constraints set by a cluster operator.\nWorkload authors don't need to know the topology of the cluster they will be running on to have their pods spread.\nBut if they do, they can still set their own spreading constraints if they have specific needs.\n\n## Motivation\n\nIn order for a workload (pod) to use `.spec.topologySpreadConstraints` (known as`PodTopologySpread`\nplugin or `EvenPodsSpreadPriority` in the old Policy API):\n\n1. Authors have to have an idea of the underlying topology.\n1. PodSpecs become less portable if their spreading constraints are tailored to a specific topology.\n\nOn the other hand, cluster operators know the underlying topology of the cluster, which makes\nthem suitable to provide default spreading constraints for all workloads in their cluster.\n\n### Goals\n\n- Cluster operators can define default spreading constraints for pods that don't provide any\n  `pod.spec.topologySpreadConstraints`.\n- Workloads are spread with the default constraints if they belong to the same service, replication controller,\n  replica set or stateful set, and if they don't define `pod.spec.topologySpreadConstraints`.\n- Provide a k8s default for `topologySpreadConstraints` that produces a priority equivalent to\n  `DefaultPodTopologySpread`, so that this plugin can be deprecated in the future.\n\n### Non-Goals\n\n- Set defaults for specific namespaces or according to other selectors.\n- Removal of `DefaultPodTopologySpread` plugin.\n\n## Proposal\n\n### User Stories\n\n#### Story 1\n\nAs a cluster operator, I want to set default spreading constraints for workloads in the cluster.\nCurrently, `SelectorSpreadPriority` provides a canned priority that spreads across nodes\nand zones (`topology.kubernetes.io/zone`). However, the nodes in my cluster have custom topology\nkeys (for physical host, rack, etc.).\n\n#### Story 2\n\nAs a workload author, I want to spread the workload in the cluster, but:\n(1) I don't know the topology of the cluster I'm running on.\n(2) I want to be able to run my PodSpec in different clusters (on-prem and cloud).\n\n### Implementation Details/Notes/Constraints\n\n\n#### Feature gate\n\nSetting a default for `PodTopologySpread` will be guarded with the feature gate\n`DefaultEvenPodsSpread`. This feature gate will depend on `EvenPodsSpread` to also be enabled.\n\n#### Relationship with \"DefaultPodTopologySpread\" plugin\n\nNote that Default `topologySpreadConstraints` has a similar effect to `DefaultPodTopologySpread`\nplugin (`SelectorSpreadingPriority` when using the Policy API).\nGiven that the latter is not configurable, they could return conflicting priorities, which\nmay not be the intention of the cluster operator or workload author. On the other hand, a proper\ndefault for `topologySpreadConstraints` could provide the same score as\n`DefaultPodTopologySpread`. Thus, there's no need for the features to co-exist.\n\nWhen the feature gate is enabled:\n\n- K8s will set Default `topologySpreadConstraints` and remove `DefaultPodTopologySpread` from the\nk8s providers (`DefaultProvider` and `ClusterAutoscalerProvider`). The\n[Default](#default-constraints) will have a similar effect.\n- When a policy is used, `SelectorSpreadingPriority` will map to `PodTopologySpread`.\n- When setting plugins in the Component Config API, plugins are added as requested. Since an\n  operator is manually enabling the plugins, we assume they are aware of their intentions.\n\n### Risks and Mitigations\n\nThe `PodTopologySpread` plugin has some overhead compared to other plugins. We currently ensure that\npods that don't use the feature get minimally affected. After Default `topologySpreadConstraints`\nis rolled out, all pods will run through the plugin.\nWe should ensure that the running overhead is not significantly higher than\n`DefaultPodTopologySpread` with the k8s Default.\n\n## Design Details\n\n### API\n\nA new structure `Args` is introduced in `pkg/scheduler/framework/plugins/podtopologyspread`.\nValues are decoded from the `pluginConfig` slice in the kube-scheduler Component Config and used in\n`podtopologyspread.New`.\n\n```go\n// pkg/scheduler/framework/plugins/podtopologyspread/plugin.go\ntype Args struct {\n\t// DefaultConstraints defines topology spread constraints to be applied to pods\n\t// that don't define any in `pod.spec.topologySpreadConstraints`. Pod selectors must\n\t// be empty, as they are deduced from the resources that the pod belongs to\n\t// (includes services, replication controllers, replica sets and stateful sets). \n\t// If not specified, the scheduler applies the following default constraints:\n\t// \u003cdefault rules go here. See next section\u003e\n\t// +optional\n\tDefaultConstraints []corev1.TopologySpreadConstraint\n}\n```\n\nNote the use of `k8s.io/api/core/v1.TopologySpreadConstraint`. During validation, we verify that\nselectors are not defined.\n\n### Default constraints\n\nThese will be the default constraints for the cluster when the operator doesn't provide any:\n\n```yaml\ndefaultConstraints:\n  - maxSkew: 1\n    topologyKey: \"kubernetes.io/hostname\"\n    whenUnsatisfiable: ScheduleAnyway\n  - maxSkew: 1\n    topologyKey: \"topology.kubernetes.io/zone\"\n    whenUnsatisfiable: ScheduleAnyway\n```\n\n### How user stories are addressed\n\nLet's say we have a cluster that has a topology based on physical hosts and racks.\nThen, an operator can set the following configuration for the plugin:\n\n```yaml\ndefaultConstraints:\n  - maxSkew: 5\n    topologyKey: \"example.com/topology/physical_host\"\n    whenUnsatisfiable: ScheduleAnyway\n  - maxSkew: 15\n    topologyKey: \"example.com/topology/rack\"\n    whenUnsatisfiable: DoNotSchedule\n```\n\nThen, a workload author could have the following `ReplicaSet`:\n\n```yaml\napiVersion: apps/v1\nkind: ReplicaSet\nmetadata:\n  name: replicated_demo\nspec:\n  replicas: 3\n  selector:\n    matchLabels:\n      app: demo\n  template:\n    metadata:\n      labels:\n        app: demo\n    spec:\n      containers:\n      - name: php-redis\n        image: example.com/registry/demo:latest\n```\n\nNote that the workload author didn't provide spreading constraints in the `pod.spec`.\nThe following spreading constraints will be derived from the constraints defined in ComponentConfig,\nand will be applied at runtime:\n\n```yaml\ntopologySpreadConstraints:\n  - maxSkew: 5\n    topologyKey: \"example.com/topology/physical_host\"\n    whenUnsatisfiable: ScheduleAnyway\n    selector:\n      matchLabels:\n        app: demo\n  - maxSkew: 15\n    topologyKey: \"example.com/topology/rack\"\n    whenUnsatisfiable: DoNotSchedule\n    selector:\n      matchLabels:\n        app: demo\n```\n\nPlease note that these constraints are honored internally in the scheduler, but they are NOT\npersisted in the PodSpec via API Server.\n\n### Implementation Details\n\n#### In the metadata/predicates/priorities flow\n\n1. Calculate the spreading constraints for the pod as part of the metadata calculation.\n   Use the constraints provided by the pod or calculate the default ones if they don't provide any.\n1. When running the predicates or priorities, use the constraints stored in the metadata.\n\n#### In the scheduler framework\n\n1. Calculate spreading constraints for the pod in the `PreFilter` extension point. Store them\n   in the `PluginContext`.\n1. In the `Filter` and `Score` extension points, use the stored spreading constraints instead of\n   the ones defined by the pod.\n\n### Test Plan\n\nTo ensure this feature to be rolled out in high quality. Following tests are mandatory:\n\n- **Unit Tests**: All core changes must be covered by unit tests.\n- **Integration Tests**: One integration test for the default rules and one for custom rules.\n- **Benchmark Tests**: A benchmark test that compare the default rules against `SelectorSpreadingPriority`.\n  The performance should be as close as possible.\n\n### Graduation Criteria\n\n#### Alpha (v1.18):\n\n- [ ] Args struct for `podtopologyspread.New`.\n- [ ] Defaults and validation.\n- [ ] Score extension point implementation.\n- [ ] Filter extension point implementation.\n- [ ] Disabling `DefaultPodTopologySpread` when the feature is enabled.\n- [ ] Test cases mentioned in the [Test Plan](#test-plan).\n\n## Implementation History\n\n- 2019-09-26: Initial KEP sent out for review.\n- 2020-01-20: KEP updated to make use of framework's PluginConfig.\n\n## Alternatives\n\n- Make the topology keys used in `SelectorSpreadingPriority` configurable.\n\n    While this moves the scheduler in the right direction, there are two problems:\n    \n    1. We can only support one topology key.\n    1. It makes it hard for pods to override the operator-provided spreading rules.\n\n- Implement a mutating controller that sets defaults.\n\n  This approach would likely allow us to provide a more flexible interface that\n  can set defaults for specific namespaces or with other selectors. However, that\n  wouldn't allow us to replace `SelectorSpreadingPriority` with\n  `EvenPodsSpreading`.\n"
+  },
+  {
+    "id": "643b86f775af2a91998deb4d6807a8d5",
+    "title": "Graduate ScheduleDaemonSetPods to GA",
+    "authors": ["@draveness"],
+    "owningSig": "sig-scheduling",
+    "participatingSigs": ["sig-apps"],
+    "reviewers": ["@k82cn", "@janetkuo"],
+    "approvers": ["@k82cn"],
+    "editor": "TBD",
+    "creationDate": "2019-10-11",
+    "lastUpdated": "2019-10-11",
+    "status": "implemented",
+    "seeAlso": [
+      "https://docs.google.com/document/d/10Ch3dhD88mnHYTq9q4jtX3e9e6gpndC78g5Ea6q4JY4/edit#heading=h.dtxm02f9bgaw"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Graduate ScheduleDaemonSetPods to GA\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n    - [Existing Tests](#existing-tests)\n  - [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [x] KEP approvers have set the KEP status to `implementable`\n- [x] Design details are appropriately documented\n- [x] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [x] Graduation criteria is in place\n- [x] \"Implementation History\" section is up-to-date for milestone\n- [x] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [x] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nScheduleDaemonSetPods has been created in the past to schedule DaemonSet Pods by the default scheduler. We wish to graduate ScheduleDaemonSetPods feature to make scheduling decisions only in the scheduler and remove the scheduling related code in the DaemonSetController.\n\n## Motivation\n\nScheduleDaemonSetPods has been beta'ed in 1.12.\n\n### Goals\n\n+ Plan to promote ScheduleDaemonSetPods to the stable version.\n+ Remove scheduling related codes in DaemonSetController.\n\n### Non-Goals\n\n+ Changing API field or meaning\n\n## Proposal\n\n### Implementation Details/Notes/Constraints\n\nScheduleDaemonSetPods attaches node affinity to DaemonSet pods, which let the default kubernetes scheduler to schedule pods on specific nodes. The DaemonSet controller uses several scheduler predicates to calculate the nodes which need to schedule DaemonSet pods and create pods with specific NodeAffinity.\n\n### Risks and Mitigations\n\nThe major concern for graduating ScheduleDaemonSetPods to the stable version could be the overhead to the scheduler and the startup time of daemons. After we graduate this feature, the scheduler would select nodes for all of the DaemonSet pods, which may cause a lot of pods with NodeAffinity to be processed.\n\n## Design Details\n\n### Test Plan\n\n#### Existing Tests\n\nScheduleDaemonSetPods currently has multiple tests in various components that use the feature.\n\n### Graduation Criteria\n\n**Note:** *Section not required until targeted at a release.*\n\n- [x] Graduate ScheduleDaemonSetPods to GA\n- [x] Remove suspenedDaemonPods which handles Pod deleted on the nodes\n- [x] Refactor nodeShouldRunDaemonPod to remove useless return values\n- [x] Update documents to reflect the changes\n\n## Implementation History\n\n+ ScheduleDaemonSetPods was introduced in Kubernetes 1.11 as an alpha version.\n+ ScheduleDaemonSetPods was graduated to beta in Kubernetes 1.12.\n"
+  },
+  {
+    "id": "6c4cbe08d28e0171e4de5ecf4e5cd62b",
+    "title": "Graduate TaintNodeByCondition to GA",
+    "authors": ["@draveness"],
+    "owningSig": "sig-scheduling",
+    "participatingSigs": ["sig-node"],
+    "reviewers": ["TBD", "@k82cn"],
+    "approvers": ["TBD", "@k82cn"],
+    "editor": "TBD",
+    "creationDate": "2019-10-12",
+    "lastUpdated": "2019-10-12",
+    "status": "implemented",
+    "seeAlso": [
+      "https://github.com/kubernetes/community/blob/master/contributors/design-proposals/scheduling/taint-node-by-condition.md"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Graduate TaintNodeByCondition to GA\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n    - [Existing Tests](#existing-tests)\n  - [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [x] KEP approvers have set the KEP status to `implementable`\n- [x] Design details are appropriately documented\n- [x] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [x] Graduation criteria is in place\n- [x] \"Implementation History\" section is up-to-date for milestone\n- [x] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [x] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nTaintNodeByCondition has been created in the past to taint node by their conditions. We wish to graduate TaintNodeByCondition feature to make scheduling decisions based on taints instead of node conditions in the scheduler.\n\n## Motivation\n\nTaintNodeByCondition has been beta'ed in 1.12.\n\n### Goals\n\n+ Plan to promote TaintNodeByCondition to the stable version.\n\n### Non-Goals\n\n+ Changing API field or meaning\n\n## Proposal\n\n### Implementation Details/Notes/Constraints\n\nTaintNodeByCondition add taints to nodes based on their conditions in the node lifecycle controller. And it could help the default scheduler to not schedule on specific nodes unless they could tolerate them.\n\nThe scheduler will remove condition-based predicates after TaintNodeByCondition was graduated to a stable version. \n\n## Design Details\n\n### Test Plan\n\n#### Existing Tests\n\nTaintNodeByCondition currently has multiple tests in various components that use the feature.\n\n+ [Admission plugin node taint tests](https://github.com/kubernetes/kubernetes/blob/cf9203501e80ecf4611e3e762a03f009d4aac6b2/plugin/pkg/admission/nodetaint/admission_test.go#L34-L121)\n+ [Kubelet TestRegisterWithApiServerWithTaint](https://github.com/kubernetes/kubernetes/blob/cf9203501e80ecf4611e3e762a03f009d4aac6b2/pkg/kubelet/kubelet_node_status_test.go#L1959-L2005)\n+ Scheduler integration tests\n  + [Daemonset tests](https://github.com/kubernetes/kubernetes/blob/cf9203501e80ecf4611e3e762a03f009d4aac6b2/test/integration/daemonset/daemonset_test.go#L966)\n  + [Taint tests](https://github.com/kubernetes/kubernetes/blob/cf9203501e80ecf4611e3e762a03f009d4aac6b2/test/integration/scheduler/taint_test.go#L69)\n+ [Daemon controller tests](https://github.com/kubernetes/kubernetes/blob/cf9203501e80ecf4611e3e762a03f009d4aac6b2/pkg/controller/daemon/daemon_controller_test.go#L1782)\n\n### Graduation Criteria\n\n**Note:** *Section not required until targeted at a release.*\n\n- [x] Graduate TaintNodeByCondition to GA\n- [x] Update documents to reflect the changes\n\n## Implementation History\n\n+ TaintNodeByCondition was introduced in Kubernetes 1.8 as an alpha version.\n+ TaintNodeByCondition was graduated to beta in Kubernetes 1.12.\n"
+  },
+  {
+    "id": "4b3d0c3919febf2b543f7910812debab",
+    "title": "Multi Scheduling Profiles",
+    "authors": ["@alculquicondor", "@ahg-g"],
+    "owningSig": "sig-scheduling",
+    "participatingSigs": null,
+    "reviewers": ["@Huang-Wei", "@liggitt"],
+    "approvers": ["@Huang-Wei"],
+    "editor": "TBD",
+    "creationDate": "2020-01-14",
+    "lastUpdated": "2020-01-14",
+    "status": "provisional",
+    "seeAlso": [
+      "/keps/sig-scheduling/20180409-scheduling-framework.md",
+      "/keps/sig-scheduling/20190226-default-even-pod-spreading.md"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Multi Scheduling Profiles\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Story 1](#story-1)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n    - [Component Config API](#component-config-api)\n      - [Conversion between API versions](#conversion-between-api-versions)\n      - [Defaults](#defaults)\n      - [Validation](#validation)\n      - [CLI flags binding](#cli-flags-binding)\n    - [Kube-Scheduler implementation](#kube-scheduler-implementation)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nAs workloads in clusters become more heterogeneous, it is natural that they have\ndifferent scheduling needs.\n\nWe propose making the scheduler run different framework plugin configurations,\nwhich we will call profiles and will be associated to a scheduler name.\nPods can choose to be scheduled under a particular configuration by setting the\nscheduler name associated to it in its pod spec. They will continue to be\nscheduled under the default configuration if they don't specify a scheduler\nname (i.e. `.spec.schedulerName`). The scheduler will continue to schedule one\npod at a time.\n\n## Motivation\n\nClusters run a variety of workloads, which can be *broadly* classified as\nservices (long-running) and batch jobs (run-to-completion). Some users may\nchoose to run only one class of workloads in a cluster, so they can provide a\nreasonable configuration that suits their scheduling needs.\n\nHowever, users may choose to run more heterogeneous workloads in a single\ncluster. Or they could have a set of fixed nodes and a set that auto-scales,\nrequiring different scheduling behaviors in each of them.\n\nPods can influence scheduling decisions with features such as node/pod affinity,\ntolerations or (alpha) even pod spreading. But there are 2 problems:\n\n- A single kube-scheduler configuration will weigh scores in such a way that\n  doesn't adjust to all types of workloads. For example, the default \n  configuration includes scores that seek high availability of services.\n- Authors of the workloads need to be aware of the cluster characteristics and\n  the weights of the scores to influence their pods' scheduling in a meaningful\n  way.\n\nTo serve such heterogeneous types of workloads, some cluster operators choose to\nrun multiple schedulers, whether those are different binaries or kube-schedulers\nwith a different configuration. But this setup might cause race conditions\nbetween the multiple schedulers, as they might have a different view of the\ncluster resources at a given time. Additionally, more binaries requires more\nmanagement effort.\n\nInstead, having a single kube-scheduler run multiple profiles will have the\nsame benefits of running multiple schedulers without running into race\nconditions.\n\n### Goals\n\n- Add support in the kube-scheduler's component config API for multiple\nscheduling profiles.\n- Make kube-scheduler schedule pods using different profiles given the scheduler\nname specified in the pod spec.\n\n### Non-Goals\n\n- Introduce new default scheduling profiles.\n\n## Proposal\n\n### User Stories\n\n#### Story 1\n\nI have two types of workloads that I want to run in two different sets of nodes.\nFor one type of workload, I want them to spread in the topology. But for the\nother type, I prefer them to get scheduled in a few nodes as possible.\n\n### Implementation Details/Notes/Constraints\n\n#### Component Config API\n\n[`v1alpha1` component config](\nhttps://github.com/kubernetes/kubernetes/blob/50deec2/pkg/scheduler/apis/config/types.go#L46)\nlooks like the following:\n\n```go\ntype KubeSchedulerConfiguration struct {\n   ...\n   SchedulerName string\n   HardPodAffinitySymmetricWeight\n   Plugins *Plugins\n   PluginConfig []PluginConfig\n   ...\n}\n```\n\nWe will introduce `v1alpha2`, with the following structure:\n\n```go\ntype KubeSchedulerConfiguration struct {\n   ...\n   Profiles []KubeSchedulerProfile\n}\n\ntype KubeSchedulerProfile struct {\n   SchedulerName string\n   Plugins *Plugins\n   PluginConfig []PluginConfig\n}\n```\n\n##### Conversion between API versions\n\nDuring conversion from `v1alpha1` to `v1alpha2`, we will copy all the necessary\nparameters from KubeSchedulerConfiguration into one item in the `Profiles` list.\n\n`HardPodAffinitySymmetricWeight` would be moved to be a `PluginConfig.Arg` in\nthe `PluginConfig` slice for the plugin `InterPodAffinity` as\n`HardPodAffinityWeight`.\n\n##### Defaults\n\nThe default configuration will look like:\n\n```yaml\nprofiles:\n  - schedulerName: 'default-scheduler'\n```\n\nNote that default plugins are loaded internally from the AlgorithmSource.\n\n`HardPodAffinityWeight` will be set to have a default of `1` in the\n`InterPodAffinity` plugin instantiation.\n\n##### Validation\n\n`SchedulerName`, `Plugins` and `PluginConfig` fields for each item in\n`Profiles` will be validated according to the same rules as `v1alpha1`. We will\nlose the early validation of `HardPodAffinitySymmetricWeight`. However, once we\ntry to instantiate a framework, the Plugin instantiation will fail, providing a\nsimilar result as the binary is starting.\n\n`SchedulerName` values will be validated to not repeat among the items of\n`Profiles`.\n\n##### CLI flags binding\n\nNote that, if component config is used, deprecated flags are currently ignored,\nwhich includes `scheduler-name` and `hard-pod-affinity-symmetric-weight`. This\nimplies that we only have to worry about these flags in relationship with the\ndefault profile.\n\nThus, if component config is not used, we will preserve the behavior of the\nflags as follows:\n- `scheduler-name` will be bound to its counterpart in the default profile.\n- `hard-pod-affinity-symmetric-weight` will be bound to a new deprecated option\n  that will be processed into a `pluginConfig` slice of the default profile,\n  like follows:\n  \n```yaml\nprofiles:\n  - schedulerName: 'default-scheduler'\n    pluginConfig:\n      - name: 'InterPodAffinity'\n      - args:\n          hadPodAffinityWeight: \u003cvalue\u003e\n```\n\n#### Kube-Scheduler implementation\n\n1. At startup, kube-scheduler will process all the different profiles,\ninitialize framework instances for them and store them in a registry.\nIf no profile is included in the configuration, one will be instantiated with\nthe name `default-scheduler` using the default plugins.\n\n2. When getting notified about unscheduled pods, kube-scheduler will check the\nscheduler name in the registry. If the name is present, they will be added to\nthe scheduler queue.\n\n3. When a new pod is taken from the queue, it will get scheduled using the\nframework instance from the registry corresponding to the specified scheduler\nname.\n\n### Risks and Mitigations\n\nOperators could introduce profiles that disable scheduling features exposed in\nthe Pod Spec. Fortunately, the framework's plugins configuration makes it easy\nto [create custom configurations from the default](\nhttps://github.com/kubernetes/kubernetes/blob/50deec2/pkg/scheduler/apis/config/types.go#L156-L160)\nthrough its `enabled` and `disabled` lists. \nHowever, we should discourage the use of `*` to disable all plugins in\nthe scheduler documentation.\n\n## Design Details\n\n### Test Plan\n\nTODO\n\n### Graduation Criteria\n\n##### Alpha -\u003e Beta Graduation\n\nTODO\n\n##### Beta -\u003e GA Graduation\n\nTODO\n\n## Implementation History\n\n- 2020-01-14: Initial KEP sent out for review, including Summary, Motivation\nand Proposal\n"
+  },
+  {
+    "id": "48cb924aa07fb80fe2e13a6e498598e7",
+    "title": "Promote Taint Based Evictions to GA",
+    "authors": ["@damemi"],
+    "owningSig": "sig-node",
+    "participatingSigs": ["sig-scheduling"],
+    "reviewers": ["@Huang-Wei"],
+    "approvers": ["sig-node", "sig-cloud-provider"],
+    "editor": "",
+    "creationDate": "2020-01-14",
+    "lastUpdated": "2020-01-14",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Promote Taint Based Evictions to GA\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Testing Plan](#testing-plan)\n  - [Unit tests](#unit-tests)\n  - [Integration tests](#integration-tests)\n  - [e2e tests](#e2e-tests)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nTaint Based Evictions was introduced as an alpha feature in Kubernetes 1.6 and was promoted to\nbeta in 1.13. The feature automatically taints nodes with `NoExecute` when they become unready or\nunreachable.\n\n## Motivation\n\nThe TaintNodesByCondition feature has worked to ensure nodes are tainted with `NoSchedule` effect\nupon different node conditions. However, it's also required to taint nodes with `NoExecute` automatically\nupon some node conditions such as node gets not ready or unreachable.\n\n### Goals\n\nEnsure nodes are tainted properly with a NoExecute effect when it's not ready or unreachable, so that\nscheduler can use taints to make scheduling decisions consistently.\n\n### Non-Goals\n\nIt is not the goal of taint based evictions to make any scheduling or removal decisions for pods, but rather\nto monitor the nodes and ensure that the proper `NoExecute` effect is applied.\n\n## Proposal\n\nEnsure test coverage is sufficient.\n\nUpdate feature gate logic around Taint Based Evictions to enable it by default.\n\nUpdate documentation to reflect the status of the feature.\n\n\n### Risks and Mitigations\n\nThere is no proposed change to the functionality of this feature, and it has functioned \nwell since its promotion to beta in 1.13, so no risks are expected.\n\n\n## Graduation Criteria\n\n* The feature has been stable and reliable in the past several releases.\n* Adequate documentation exists for the feature.\n* Test coverage of the feature is acceptable. This includes moving existing tests to be under the appropriate sigs.\n\n## Testing Plan\n\nTaint based evictions is comprised of node lifecycle functions taints and\nevictions, as well as the feature itself, all of which have stable unit, e2e,\nand integration tests that are run regularly as part of the Kubernetes CI/CD pipeline.\n\n### Unit tests\n* [Node lifecycle controller eviction tests](https://github.com/kubernetes/kubernetes/blob/47d5c3ef8d/pkg/controller/nodelifecycle/node_lifecycle_controller_test.go#L196)\n\n### Integration tests\n* [Taint based evictions integration test](https://github.com/kubernetes/kubernetes/blob/47d5c3ef8df2b1b26da739aec0ada15d41f20cf3/test/integration/scheduler/taint_test.go#L580) (note that prior to 1.17, this test existed as an [end-to-end test](https://github.com/kubernetes/kubernetes/blob/001f2cd2b553d06028c8542c8817820ee05d657f/test/e2e/scheduling/taint_based_evictions.go)\n\n### e2e tests\n* [Scheduler taints e2e tests](https://github.com/kubernetes/kubernetes/blob/master/test/e2e/scheduling/taints.go)\n\n## Implementation History\n\nThe original implementation of taint based evictions predates the KEP process, so discussion on it can be found here: https://github.com/kubernetes/enhancements/issues/166\n\n"
+  },
+  {
+    "id": "9d62860c730e2baf12d0d10b4af57619",
+    "title": "Coscheduling",
+    "authors": ["@k82cn"],
+    "owningSig": "sig-scheduling",
+    "participatingSigs": ["wg-machine-learning"],
+    "reviewers": ["@bsalamat", "@vishh"],
+    "approvers": ["@bsalamat"],
+    "editor": "",
+    "creationDate": "2018-07-03",
+    "lastUpdated": "2019-01-03",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Coscheduling\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Motivation](#motivation)\n- [Function Detail](#function-detail)\n  - [API Definition](#api-definition)\n  - [Lifecycle Management](#lifecycle-management)\n  - [Scheduling](#scheduling)\n  - [Customized Controller](#customized-controller)\n- [Feature Interaction](#feature-interaction)\n  - [Multi-scheduler](#multi-scheduler)\n  - [Priority/Preemption](#prioritypreemption)\n  - [Pod RestartPolicy](#pod-restartpolicy)\n  - [Admission Controller](#admission-controller)\n  - [Cluster Autoscaler](#cluster-autoscaler)\n  - [Kubectl](#kubectl)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [References](#references)\n\u003c!-- /toc --\u003e\n\n## Motivation\n\nKubernetes has become a popular solution for orchestrating containerized workloads; it has been largely successful in orchestrating serving and storage workloads, and, with native K8s support for Spark. Meanwhile, the community also try to run Machine Learning (ML) workloads on Kubernetes, e.g. [kubeflow/tf-operator](https://github.com/kubeflow/tf-operator). When running a Tensorflow/MPI job, all tasks of a job must be start together; otherwise, did not start anyone of tasks. If the resource is enough to run all 'tasks', everything is fine; but it's not true for most of case, especially in the on-prem environment. In worst case, all jobs are pending here because of deadlock: every job only start part of tasks, and waits for the other tasks to start. It'll be worse in federation for cross-domain case which is out of scope in this doc.\n\nAfter the discussion at [Coscheduling/Gang-scheduling](https://docs.google.com/document/d/1AUwcvTtULNvow5M9e428FnlvINO1uQ7ojRoTGuTp4DA/edit#heading=h.ckn8nv2jj0xv) proposal, we decide to implement Coscheduling in [kube-batch](https://github.com/kubernetes-sigs/kube-batch) by CRDs. kube-batch focuses on \"batch\" workload in kubernetes, and will share the same [scheduling frameworks](https://github.com/kubernetes/community/pull/2281) when it's ready. This document is used to provide definition of API object and the scheduler behaviour of Coscheduling.\n\n## Function Detail\n\n### API Definition\n\nThe following requirements are identified during the discussion of this feature:\n\n1. Existing workload can use this feature without (or with a few) configuration changes\n2. Pods of a group/gang may have different `PodSpec` (and/or belong to different collections)\n3. Existing controllers which are responsible for managing life cycle of collections work well with this feature\n\nTo meet the requirements above, the following **Kind** is introduced by CRD under `incubator.scheduling.k8s.io/v1alpha1` **Group**/**Version**.\n\n```go\n// PodGroup defines the scheduling requirement of a pod group\ntype PodGroup struct {\n    metav1.TypeMeta\n    metav1.ObjectMeta\n\n\t// Spec defines the behavior of a pod group.\n\t// +optional\n    Spec PodGroupSpec\n\n\t// Status represents the current information about a pod group.\n\t// This data may not be up to date.\n\t// +optional\n    Status PodGroupStatus\n}\n\n// PodGroupSpec represents the template of a pod group.\ntype PodGroupSpec struct {\n    // MinMembers defines the minimal number of members/tasks to run the pod group;\n    // if there's not enough resources to start all tasks, the scheduler\n    // will not start anyone.\n    MinMembers int\n\n    // TotalResources defines the total resource the PodGroup requests to run\n    // Pods.\n    TotalResources v1.ResourceList\n}\n\n// PodGroupStatus represents the current state of a pod group.\ntype PodGroupStatus struct {\n    // The number of admitted pods.\n    // +optional\n    Admitted int32\n\n    // The number of actively running pods.\n    // +optional\n    Running int32\n\n    // The number of pods which reached phase Succeeded.\n    // +optional\n    Succeeded int32\n\n    // The number of pods which reached phase Failed.\n    // +optional\n    Failed int32\n}\n```\n\nThe `PodGroup`, which is a namespaced object, specifies the attributes and status of a pod group, e.g. number of pods in a group. To define which pods are member of `PodGroup`, the following annotation key is introduced for `Pod`; the annotation key is used for this alpha feature, and it'll be changed to a more permanent form, such a field, when moving `PodGroup` to core.\n\n```go\nscheduling.k8s.io/group-name\n```\n\nThe `scheduling.k8s.io/group-name` annotation specifies the `PodGroup` that it belongs to; and the pod can only belong to the `PodGroup` in the same namespace. The pod, controlled by different collections, can also belong to the same `PodGroup`.  Because of performance concern,  it does not use `LabelSelector` to build the relationship between `PodGroup` and `Pod`.\n\n### Lifecycle Management\n\nAs the lifecycle of Pods in PodGroup may be different from controller to another, the lifecycle of the members is not managed by the coscheduling feature. Each collection controller may implement or already have the mean to manage lifecycle of its members. The scheduler'll record related events for controller to manage pods, e.g. `Unschedulable`. A controller of `PodGroup` will be introduced later for lifecycle management, e.g. restart the whole `PodGroup`, according to the configuration, when the number of running pods drop below `spec.MinMembers` at run-time.\n\nThe update to `PodGroup` is not supported for now; and deleting `PodGroup` does not impact Pod's status.\n\n### Scheduling\n\nThe scheduler only watches `PodGroup` and `Pod`. It'll reconstruct 'Job' by annotation of Pod and `PodGroup`, the `Pod`s are considered as 'Task' of 'Job'; if annotation is empty, the scheduler records an unschedulable event of pod to ask user/controller to resubmit it. The schduler does not schedule pods until its `PodGroup` is created.\n\nAs batch scheduler and default scheduler may be running in parallel; the batch scheduler follows multi-scheduler feature to only handle the `Pod` that submitted to it. The batch scheduler does scheduling as follow:\n\n1. Reconstructing 'Job' by the annotation of `Pod` and `PodGroup`\n2. If there are less Pods than `minMembers` of `PodGroup`, the 'job' will not be scheduled; and an unschedulable event of `Pod` will be recorded\n3. In `allocate` phase, scheduler will\n   * record an `Unschedulable` event of `PodGroup` if some pods are running but `succeeded + pending + running \u003c minMembers`, the controller takes action according to its configuration\n   * allocate (but not bind) resource to Pods according to Pod's spec, e.g. `NodeAffinity`\n   * bind all Pods to hosts until job is ready: if `minMembers` \u003c= `allocated Pods` + `pending Pods`, it's ready when `minMembers` \u003c= `allocated Pods`; otherwise, `numMember` \u003c= `allocated Pods` + `succeeded Pods`\n4. If can not allocate enough resources to the job, the pods stay pending; and the resource cannot be allocated to other job\n\nThat may make resources (less than job's resource request) idle for a while, e.g. a huge job. The solution, e.g. backfill other smaller jobs to improve the resource utilization, will be proposed in coming release. In `allocate` phase, only pod's `NodeAffinity` takes effect; the other predicates/priorities will be included on-demand in coming release.\n\n### Customized Controller\n\nA typical example of customized controller is [kubeflow/tf-operator](https://github.com/kubeflow/tf-operator), which managed the Pods for TensorFlow on Kubernetes, required `gang-scheduling` in upstream. Here's an example of customized controller that demonstrated the usage of  `gang-scheduling` in `kube-batch`.\n\nUsually, CRD ([CustomResourceDefinitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/)) feature is used to introduce a customized **Kind**, named `CRDJob` as example. The customized controller, named `CRDJobController`, watches it and manages lifecycle of it:\n\n1. For each `CRDJob`, `CRDJobController` creates a `CRDJob` and `PodGroup` (one `CRDJob` with one `PodGroup` as example). The attributes of `PodGroup`should be set accordingly, e.g `numMember`; it's up to customized controller on how to manage relationship between `PodGroup` and `CRDJob`, e.g. `metadata.name`.\n2. When `CRDJobController` create Pods, its annotation should be set accordingly. `kube-batch` follows gang-scheduling logic to schedule those pods in batch.\n3. When pods failed/deleted/unschedulable, it is up to `CRDJobController` on how to manage `CRDJob`'s lifecycle. For example, if `CRDJobController` manages lifecycle itself, set `.spec.Policy` of `PodGroup` to nil; otherwise, `PodGroupController` will manage the lifecycle as described above.\n4. If `CRDJob` was deleted, the `PodGroup` must be deleted accordingly.\n\n## Feature Interaction\n\n### Multi-scheduler\n\nSince multiple schedulers work in parallel, there may be decision conflict between different schedulers; and the kubelet will reject one pod (failed) if conflict. The controller will handle rejected pods based on its lifecycle policy for failed pods. Users and cluster admins may reduce the probability of such conflicts by partitioning the clusters logically, for example, by placing node-affinity to distinct set of nodes on various groups of pods. \n\n### Priority/Preemption\n\nA rejected or preempted batch/run-to-completion pod may trigger a restart of the whole `PodGroup`. This can have negative impact on performance.  The solution on how to handle conflicts better will be proposed in coming release.\n\nThe default scheduler should also consider `PodGroup` when preempting pods, similar to `PodDisruptionBudgets`.\n\n### Pod RestartPolicy\n\nPod's `RestartPolicy` still works as before. But for batch/run-to-compelete workload, it's better to set `RestartPolicy` to `Never` to avoid endless restart loop.\n\n### Admission Controller\n\nIf quota runs out in the middle of creating a group of pods, a few members of a `PodGroup` may be created, while the rest will be denied by the `ResourceQuota` admission controller. `.spec.TotalResource` is added in `PodGroup` to address this problem. When a `PodGroup` is created with `.spec.TotalResource`, so much quota is reserved for the group if there is available quota. Pods of group use the already reserved quota. By setting `.spec.TotalResource` properly, one can ensure that Pods of a `PodGroup` have enough quota at creation time. The design on `Quota` enhancement to support `.spec.TotalResource` will be proposed later for review.\n\n### Cluster Autoscaler\n\n[Cluster Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) is a tool that\nautomatically adjusts the size of the Kubernetes cluster when one of the following conditions is true:\n\n* there are pods that failed to run in the cluster due to insufficient resources,\n* there are nodes in the cluster that have been underutilized for an extended period of time and their pods can be placed on other existing nodes.\n\nWhen Cluster-Autoscaler scale-out a new node, it leverage predicates in scheduler to check whether the new node can be\nscheduled. But Coscheduling is not an implementation of predicates for now; so it'll not work well together with\nCluster-Autoscaler right now. Alternative solution will be proposed later for that.\n\n### Kubectl\n\nkubectl is enhanced to support `PodGroup` by [kubectl plugins](https://kubernetes.io/docs/tasks/extend-kubectl/kubectl-plugins/), including its status.\n\n## Graduation Criteria\n\nCoscheduling is going to be implemented in [kube-batch](https://github.com/kubernetes-sigs/kube-batch/);\nwe'll consider to migrate when the above features are finished and stable. At least the following two features\nare stable enough: \n\n- Coscheduling based on `PodGroup`\n- Support `.spec.TotalResource` \n\n## Implementation History\n\n1. `PodGroup` CRD and Coscheduling (kube-batch: [v0.3](https://github.com/kubernetes-sigs/kube-batch/releases/tag/v0.3))\n1. Admission controller for `.spec.TotalResource` (kube-batch: v0.4, v0.5)\n1. `PodGroup` controller (kube-batch: v0.4, v0.5)\n\n## References\n\n* [Coscheduling in Kubernetes](https://docs.google.com/document/d/1AUwcvTtULNvow5M9e428FnlvINO1uQ7ojRoTGuTp4DA/edit#heading=h.ckn8nv2jj0xv)\n* [Indexed Job](https://github.com/kubernetes/kubernetes/issues/14188)\n* [Schedule a group of pods all at once](https://github.com/kubernetes/kubernetes/issues/16845)\n* [kubeflow/tf-operator: Prevent scheduling deadlocks](https://github.com/kubeflow/tf-operator/issues/165)\n* [Added PodGroup Phase in Status](https://github.com/kubernetes-sigs/kube-batch/pull/533)\n"
+  },
+  {
+    "id": "2863f4be49c983501535478745aadbe0",
+    "title": "Resource Quota based on Node Labels",
+    "authors": ["@vishh", "@bsalamat"],
+    "owningSig": "sig-scheduling",
+    "participatingSigs": ["sig-architecture"],
+    "reviewers": ["@derekwaynecarr", "@davidopp"],
+    "approvers": ["TBD"],
+    "editor": "TBD",
+    "creationDate": "2018-08-23",
+    "lastUpdated": "2018-08-30",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Resource Quota based on Node Labels\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n- [Potential solutions](#potential-solutions)\n    - [Pros](#pros)\n    - [Cons](#cons)\n    - [Pros](#pros-1)\n    - [Cons](#cons-1)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nAllowing Resource Quota to be applied on pods based on their node selector configuration opens up a flexible interface for addressing some immediate and potential future use cases.\n\n## Motivation\n\nAs a kubernetes cluster administrator, I'd like to,\n\n1. Restrict namespaces to specific HW types they can consume. Nodes are expected to be homogeneous wrt. to specific types of HW and HW type will be exposed as node labels.\n   * A concrete example - An intern should only use the cheapest GPU available in my cluster, while researchers can consume the latest or most expensive GPUs.\n2. Restrict compute resources consumed by namespaces on different zones or dedicated node pools.\n3. Restrict compute resources consumed by namespaces based on policy (FIPS, HIPAA, etc) compliance on individual nodes.\n\nThis proposal presents flexible solution(s) for addressing these use cases without introducing much additional complexity to core kubernetes.\n\n## Potential solutions\n\nThis proposal currently identifies two possible solutions, with the first one being the _preferred_ solution.\n\n### Solution A - Extend Resource Quota Scopes\n\nResource Quota already includes a built in extension mechanism called [Resource Scopes](https://github.com/kubernetes/api/blob/master/core/v1/types.go#L4746).\nIt is possible to add a new Resource Scope called “NodeAffinityKey” (or something similar) that will allow for Resource Quota limits to apply to node selector and/or affinity fields specified in the pod spec.\n\nHere’s an illustration of a sample object with these new fields:\n\n```yaml\napiVersion: v1\nkind: ResourceQuota\nmetadata:\n  name: hipaa-nodes\n  namespace: team-1\nspec:\n  hard:\n    cpu: 1000\n    memory: 100Gi\n  scopeSelector:\n   scopeName: NodeAffinityKey\n   operator: In\n   values: [“hipaa-compliant: true”] \n```\n\n``` yaml\napiVersion: v1\nkind: ResourceQuota\nmetadata:\n  name: nvidia-tesla-v100-quota\n  namespace: team-1\nspec:\n  hard:\n   - nvidia.com/gpu: 128\n  scopeSelector:\n   scopeName: NodeAffinityKey\n   operator: In\n   values: [“nvidia.com/gpu-type:nvidia-tesla-v100”]\n```\n\nIt is possible for quotas to overlap with this feature as is the case today.\nAll quotas have to be satisfied for the pod to be admitted.\n\n[Quota configuration object](https://github.com/kubernetes/kubernetes/blob/7f23a743e8c23ac6489340bbb34fa6f1d392db9d/plugin/pkg/admission/resourcequota/apis/resourcequota/types.go#L32) will also support the new scope to allow for preventing pods from running on nodes that match a label selector unless a corresponding quota object has been created.\n\n#### Pros\n\n- Support arbitrary properties to be consumed as part of quota as long as they are exposed as node labels.\n- Little added cognitive burden - follows existing API paradigms.\n- Implementation is straightforward.\n- Doesn’t compromise portability - Quota remains an administrator burden.\n\n#### Cons\n\n- Requires property labels to become standardized if portability is desired. This is required anyways irrespective of how they are exposed outside of the node for scheduling portability.\n- Label keys and values are concatenated. Given that most selector use cases for quota will be deterministic (one -\u003e one), the proposed API schema might be adequate.\n\n### Solution B - Extend Resource Quota to include an explicit Node Selector field\n\nThis solution is similar to the previous one with changes to the API where instead of re-using scopes we can add an explicit Node Selector field to the Resource Quota object.\n\n```yaml\napiVersion: v1\nkind: ResourceQuota\nmetadata:\n  name: hipaa-nodes\n  namespace: team-1\nspec:\n  hard:\n   cpu: 1000\n   memory: 100Gi\n  podNodeSelector:\n   matchExpressions:\n    - key: hipaa-compliant\n      operator: In\n      values: [\"true\"]\n```\n\nUsers should already be familiar with the Node Selector spec illustrated here as it is used in pod and volume topology specifications.\nHowever this solution introduces a field that is only applicable to a few types of resources that Resource Quota can be used to control.\n\n### Solution C - CRD for expressing Resource Quota for extended resources\n\nThe idea behind this solution is to let individual kubernetes vendors create additional CRDs that will allow for expressing quota per namespace for their resource and have a controller that will use mutating webhooks to quota pods on creation \u0026 deletion.\nThe controller can also keep track of “in use” quota for the resource it owns similar to the built in resource quota object.\nThe schema for quota is controlled by the resource vendor and the onus of maintaining compatibility and portability is on them.\n\n#### Pros\n\n- Maximum flexibility\n  - Use arbitrary specifications associated with a pod to define quota policies\n  - The spec for quota itself can be arbitrarily complex\n- Develop and maintain outside of upstream\n\n#### Cons\n\n- Added administrator burden. An admin needs to identify multiple types of quota objects based on the HW they consume.\n- It is not trivial to develop an external CRD given the lack of some critical validation, versioning, and lifecycle primitives.\n- Tracking quota is non trivial - perhaps a canonical (example) quota controller might help ease the pain.\n- Hard to generate available and in-use quota reports for users - existing quota support in ecosystem components will not support this new quota object (kubectl for example).\n"
+  },
+  {
+    "id": "e21fe5394338414bf7ea6dcbfa5f69a8",
+    "title": "Volume Subpath Env Expansion",
+    "authors": ["@kevtaylor"],
+    "owningSig": "sig-storage",
+    "participatingSigs": ["sig-storage", "sig-architecture"],
+    "reviewers": ["@msau42", "@thockin", "@liggitt"],
+    "approvers": ["@liggitt", "@msau42"],
+    "editor": "TBD",
+    "creationDate": "2018-10-29",
+    "lastUpdated": "2018-03-04",
+    "status": "implementable",
+    "seeAlso": ["n/a"],
+    "replaces": ["n/a"],
+    "supersededBy": ["n/a"],
+    "markdown": "\n# Volume Subpath Env Expansion\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nLegacy systems create all manner of log files and these are not easily streamed into stdout\n\nFiles that are written to a host file path need to be uniquely partitioned\n\nIf 2 or more pods run on the same host writing the same log file names to the same volume, they will clash\n\nUsing the `subPath` is a neat option but because the `subPath` is \"hardcoded\" ie. `subPath: mySubPath` it does not enforce uniqueness\n\nTo alleviate this, the `subPath` should be able to be configured from an environment variable as `subPath: $(MY_VARIABLE)` \n\nThe workaround to this issue is to use symbolic links or relative symbolic links but these introduce sidecar init containers\nand a messy configuration overhead to try to create upfront folders with unique names - examples of this complexity are detailed below\n\n## Motivation\n\nThe initial alpha feature was implemented to allow unique addressing of subpaths on a host\nThis cannot currently be achieved with the downwardAPI and requires complex workarounds\n\nThe workarounds became more difficult after 1.9.3 when symbolic links were removed from initContainers\n\n### Goals\n\nTo reduce excessive boiler-plate workarounds and remove the need for complex initContainers\n\n### Non-Goals\n\nFull template implementation for subPaths\n\n## Proposal\n\nThe api change proposed is to create a Mutually Exclusive Field separate from the `subPath`\ncalled `subPathExpr`\n\nThe subpath code which expands environment variables from the API would (under this proposal) change from\n\n```\n    env:\n    - name: POD_NAME\n      valueFrom:\n        fieldRef:\n          apiVersion: v1\n          fieldPath: metadata.name\n \n   ...\n   \n    volumeMounts:\n    - name: workdir1\n      mountPath: /logs\n      subPath: $(POD_NAME)\n```\n\nto:\n\n```\n    volumeMounts:\n    - name: workdir1\n      mountPath: /logs\n      subPathExpr: $(POD_NAME)\n```\n\nThis would then introduce the new element to be processed separately from the `subPath`\n\n### User Stories\n\n## Current workarounds - k8s \u003c=1.9.3\n\nThis makes use of symbolical linking to the underlying subpath system\nThe symbolic link element was removed after 1.9.3\n\n```\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n  labels:\n    app: podtest\n  name: podtest\nspec:\n  replicas: 1\n  selector:\n    matchLabels:\n      app: podtest\n  template:\n    metadata:\n      labels:\n        app: podtest\n    spec:\n      containers:\n      - env:\n        - name: POD_NAME\n          valueFrom:\n            fieldRef:\n              apiVersion: v1\n              fieldPath: metadata.name\n        - name: POD_NAMESPACE\n          valueFrom:\n            fieldRef:\n              apiVersion: v1\n              fieldPath: metadata.namespace\n        image: \u003cimage\u003e\n        name: podtest\n        volumeMounts:\n        - mountPath: /logs\n          name: workdir\n          subPath: logs\n      initContainers:\n      - command:\n        - /bin/sh\n        - -xc\n        - |\n          LOGDIR=/logs/${POD_NAMESPACE}/${POD_NAME}; mkdir -p ${LOGDIR} \u0026\u0026 ln -sfv ${LOGDIR} /workdir/logs \u0026\u0026 chmod -R ugo+wr ${LOGDIR}\n        env:\n        - name: POD_NAME\n          valueFrom:\n            fieldRef:\n              apiVersion: v1\n              fieldPath: metadata.name\n        - name: POD_NAMESPACE\n          valueFrom:\n            fieldRef:\n              apiVersion: v1\n              fieldPath: metadata.namespace\n        image: alpine:3.5\n        name: prep-logs\n        volumeMounts:\n        - mountPath: /logs\n          name: logs\n        - mountPath: /workdir\n          name: workdir\n      volumes:\n      - emptyDir: {}\n        name: workdir\n      - hostPath:\n          path: /logs\n          type: \"\"\n        name: logs\n```\n\n## Workarounds - k8s \u003e1.9.3\n\nBeyond 1.9.3 some attempts were made to provide a workaround using relative paths, rather than symbolic links\n\nThese have been deemed to be a cumbersome manipulation of the operating system and are flawed and unworkable\n\nThis effectively negates an upgrade path to k8s 1.10\n\nThe only foreseeable solution is to move directly to 1.11 from 1.9.3 and switch on the alpha feature gate VolumeSubPathEnvExpansion\n\n## Alternatives - using subPath directly\n\nAn initial attempt has been made for this but there are edge case compatibility issues which are highlighted here \nhttps://github.com/kubernetes/kubernetes/pull/65769 regarding the alpha implementation \n\nThe objection is regarding backward compatibility with existing users' subpaths\n\nBecause of a breaking change in the API, it has been decided to offer an alternative based on the\noriginal discussion here https://github.com/kubernetes/kubernetes/issues/48677\n\nThe `VolumeSubPathEnvExpansion` alpha feature was delivered in k8s 1.11 allowing\nsubpaths to be created from downward api variables as \n```\napiVersion: v1\nkind: Pod\nmetadata:\n  name: pod1\nspec:\n  containers:\n  - name: container1\n    env:\n    - name: POD_NAME\n      valueFrom:\n        fieldRef:\n          apiVersion: v1\n          fieldPath: metadata.name\n    image: busybox\n    command: [ \"sh\", \"-c\", \"while [ true ]; do echo 'Hello'; sleep 10; done | tee -a /logs/hello.txt\" ]\n    volumeMounts:\n    - name: workdir1\n      mountPath: /logs\n      subPath: $(POD_NAME)\n  restartPolicy: Never\n  volumes:\n  - name: workdir1\n    hostPath: \n      path: /var/log/pods\n```\n\nBecause of the mentioned breaking changes, this implementation cannot proceed forward\n\n### Risks and Mitigations\n\nThe alpha implementation already provided a number of test cases to ensure that validations of `subPath` configurations \nwere not circumvented or violated\n\nThe API change would ensure that the substitute of the variables takes place immediately before the subpath mount validation\n\nWe would also need review existing validation to ensure that any potential security issues are addressed as:\n`$ escape and \"../../../../../proc are not allowed`\n\nDue to the vulnerabilities highlighted in https://github.com/kubernetes/kubernetes/issues/60813 the subpath validations in kubelet have been\nhighly orchestrated. Any implementation of this feature needs to ensure that the security fixes put in place are still effective\n\n\n## Graduation Criteria\n\nThe existing alpha feature introduced many tests to mitigate issues. These would be reused as part of the api implementation.\n\n[umbrella issues]: https://github.com/kubernetes/kubernetes/pull/49388\n\nThe tests that have been approved for 1.14 Alpha include: \n* Verification that a failing subpath expansion can be modified during the lifecycle of a container\n* Verification that subpaths can be written correctly\n* Verification that subpath mounts remain consistent on a container restart\n* Verification of consistency if environment variables change \n\nTests that have been operational for the initial feature gate since 1.11 include:\n* Substituting values in a volume subpath\n* Substituting values in a volume subpath with absolute path should fail\n* Substituting values in a volume subpath with backticks should fail\n\nUnless any edge cases are detected during alpha stage testing, these exercise paths through subpath mounts\nand ensure that no validation is violated and vulnerabilities are not introduced\n\nThe node alpha suite tests paths through kubelet and maintains that the lifecycle follows that of subpaths\n\nMoving Alpha-\u003eBeta should be a simple migration step in 1.15 unless any edge cases are discovered outside of alpha test suites\nBeta-\u003eGA should occur in 1.17 after a period of Beta implmentation and not uncovering any issues\n\n## Implementation History\n\n* Initial issue: https://github.com/kubernetes/kubernetes/issues/48677\n* Feature gate proposal: https://github.com/kubernetes/enhancements/issues/559\n* Alpha Implementation: https://github.com/kubernetes/kubernetes/pull/49388\n* Beta Issue: https://github.com/kubernetes/kubernetes/issues/64604\n* Beta PR and Discussion: https://github.com/kubernetes/kubernetes/pull/65769\n\nAlpha 1.14 milestones achieved:\n* Alpha implementation: https://github.com/kubernetes/kubernetes/pull/71351\n* Documentation: https://github.com/kubernetes/website/pull/11843\n* Alpha test grid: https://k8s-testgrid.appspot.com/sig-storage-kubernetes#gce-alpha-features\n* Alpha node test grid: https://k8s-testgrid.appspot.com/sig-node-kubelet#node-kubelet-alpha\n\nBeta 1.15 milestones achieved:\n* Beta implementation: https://github.com/kubernetes/kubernetes/pull/76546\n* Documentation: https://github.com/kubernetes/website/pull/13846\n* sig-storage test grid: https://testgrid.k8s.io/sig-storage-kubernetes#gce-slow\n* Node kubelet feature test grid: https://testgrid.k8s.io/sig-node-kubelet#node-kubelet-features\n\nStable 1.17 milestones achieved:\n* Stable implementation: https://github.com/kubernetes/kubernetes/pull/82578\n* Documentation: https://github.com/kubernetes/website/pull/16547\n* sig-storage test grid: https://k8s-testgrid.appspot.com/sig-storage-kubernetes#gce\u0026include-filter-by-regex=Variable\n* sig-storage slow tests: https://k8s-testgrid.appspot.com/sig-storage-kubernetes#gce-slow\u0026include-filter-by-regex=Variable\n\n## Alternatives - Using subPathFrom\nA possible further implementation could derive directly from the `fieldRef` as\n\n```\n        volumeMounts:\n        - mountPath: /logs\n          name: logs\n          subPathFrom:\n            fieldRef:\n              fieldPath: metadata.name\n      volumes:\n      - name: logs\n        hostPath:\n          path: /logs\n```\n\nThis method would not be favoured as it fixes the `subPath` to a single value and would not allow concatenation\nof paths such as `$(NAMESPACE)/$(POD_NAME)`\n\n\n\n"
+  },
+  {
+    "id": "6c0d08f3629ee4e722b8cb9137bbf473",
+    "title": "Extend usage of Volume DataSource to allow PVCs for Cloning",
+    "authors": ["@j-griffith"],
+    "owningSig": "sig-storage",
+    "participatingSigs": ["sig-architecture"],
+    "reviewers": ["TBD"],
+    "approvers": ["@saad-ali", "@thockin"],
+    "editor": "@j-griffith",
+    "creationDate": "2018-11-11",
+    "lastUpdated": "2019-07-17",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Allow the use of the dataSource field for clones (existing PVCs)\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories [optional]](#user-stories-optional)\n    - [Story 1](#story-1)\n    - [Story 2](#story-2)\n    - [Story 3](#story-3)\n    - [Story 4](#story-4)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Drawbacks [optional]](#drawbacks-optional)\n- [Alternatives [optional]](#alternatives-optional)\n- [Infrastructure Needed [optional]](#infrastructure-needed-optional)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis KEP proposes adding support for specifying existing PVCs in the DataSource field to indicate a user would like to Clone a Volume.  Note that this KEP also applies ONLY to dynamic provisioner, and ONLY CSI Provisioner's.\n\nFor the purpose of this KEP, a Clone is defined as a duplicate of an existing Kubernetes Volume that can be consumed as any standard Volume would be.  The only difference is that upon provisioning, rather than creating a \"new\" empty Volume, the back end device creates an exact duplicate of the specified Volume.\n\nClones are different than Snapshots. A Clone results in a new, duplicate volume being provisioned from an existing volume -- it counts against the users volume quota, it follows the same create flow and validation checks as any other volume provisioning request, it has the same life cycle and work flow. Snapshots, on the other hand, results in a point-in-time copy of a volume that is not, itself, a usable volume -- it can be used to either to provision a new volume (pre-populated with the snapshot data) or to restore the existing volume to a previous state (represented by the snapshot).\n\n## Motivation\n\nFeatures like Cloning are common in most storage devices, not only is the capability available on most devices, it's also frequently used in various use cases whether it be for duplicating data or to use as a disaster recovery method.\n\n### Goals\n\n* Add ability to specify a PVC in a users Namespace as a DataSource\n  - Add Core PVC Object to the permitted DataSource types to API Validation\n* Provide ability to pass Clone intent to a CSI Plugin that reports it supports Clone capability\n  - Proposal is limited to allowing a user to specify a Clone request, and for that Clone request to be passed to CSI Plugins that report support for cloning via capabilities\n\n### Non-Goals\n\n* This KEP does NOT propose the addition of other types of DataSource including Populators\n* This KEP does NOT propose support for special cases like \"out of band\" cloning (support for back ends that don't have Cloning features), that sort of implementation would fall under Populators.\n* This KEP does NOT propose any ability to shrink a PVC during a Clone request (e.g. it's considered an invalid request to clone PVC-a with a size of 10Gib to a PVC with a requested size of less than 10Gib, expansion is \"ok\" if the driver supports it but it's not required)\n* This KEP does NOT propose adding any ability to transfer a Clone to a different Namespace, the new PVC (Clone) will be in the same Namespace as the origin that was specified.  This also means that since this is namespaced, a user can not request a Clone of a PVC that is another Namespace.  A user can only request a Clone of PVCs in his or her Namespace.\n* Cloning will only be available in CSI, cloning features will NOT be added to existing in-tree plugins or Flex drivers\n* Cloning will only be available within the same storage class (see [Implementation Details](#implementation-detailsnotesconstraints-optional) section for more info)\n\n## Proposal\n\nAdd the Core object PVC to the allowed types for PVC DataSource.  Currently API validation only allows Snapshot Object Types, this proposal is to also add the Core PersistentVolumeClaim object as an accepted DataSource entry.\n\nThe following example assumes a PVC with the name `pvc-1` exists in the Namespace `myns` and has a size less than or equal to 10Gi:\n\n```yaml\nkind: PersistentVolumeClaim\napiVersion: v1\nmetadata:\n    name: pvc-2\n    namespace: myns\nspec:\n  capacity:\n    storage: 10Gi\n  dataSource:\n    kind: PersistentVolumeClaim\n    name: pvc-1\n```\n\nThe result would be a new and independent PV/PVC (pvc-2) on the back end device that is a duplicate of the data that existed on pvc-1.  It is required that this PVC be in the same Namespace as the original (that specified by DataSource).\n\n### User Stories [optional]\n\n#### Story 1\nAs a cluster user, I want to easily test changes to my production data base without risking issues to my customer facing applications\n\n#### Story 2\nAs a cluster user, I want to be able to easily Clone a volume and run a different set of PODs/Applications against it\n\n#### Story 3\nAs a cluster user, I want to be able to easily duplicate an existing deployment that's running on my Cluster to use for testing or the next version of my application\n\n#### Story 4\nAs a cluster admin or user, I want to be able to provide the equivalent of data templates to users in the Cluster to ensure consistent and secure data sets\n\n### Implementation Details/Notes/Constraints [optional]\n\nThis proposal requires adding PersistentVolumeClaims as allowed Object Types to the API validation checks against the DataSource field.  Currently the only allowed Object Type is SnapshotDataSource, this proposal would require the addition of the Core Object PersistentVolumeClaim as well, in addition to unit tests.  In addition this would also require a feature gate specifically for the Clone option (PVCDataSource).\n\nCurrently the CSI provisioner already accepts the DataSource field in new provisioning requests.  The additional implementation that's needed is to add acceptance of PVC types in the current CSI external-provisioner.  Once that's added, the PVC info can then be passed to the CSI Plugin in the DataSource field and used to instruct the backend device to create a Clone.\n\nTo emphasize above, this feature will ONLY be available for CSI.  This feature wil NOT be added to In-tree plugins or Flex drivers, this is strictly a CSI only feature.\n\nIt's important to note that we intentionally require that a source PVC be in the same StorageClass as the PVC being created.  This is currently required because the StorageClass determintes characteristics for a volume like `fsType`.  Performing a clone from an xfs volume to an ext4 volume for example would not be acceptable, given that a storageClass can have unique information, cloning across storage classes is not something we're able to try and determine safely at this time.\n\n### Risks and Mitigations\n\nThe primary risk of this feature is requesting a PVC DataSource when using a CSI Driver that doesn't handle Cloning in a safe way for running applications.  It's assumed that the responsibility for reporting Clone Capabilities in this case is up to the CSI Driver, and if a CSI Driver is reporting Clone support that implies that they can in fact Clone Volumes without disrupting or corrupting users that may be actively using the specified source volume.\n\nDue to the similarities between Clones and Snapshots, it is possible that some back ends may require queiscing in-use volumes before cloning.  This proposal suggests that initially, if a csi plugin is unable to safely perform the requested clone operation, it's the csi plugins responsibility to reject the request.  Going forward, when execution hooks are available (currently being proposed for consistent snapshots), that same mechanism should be made generally usable to apply to Clones as well.\n\n## Graduation Criteria\n* API changes allowing specification of a PVC as a valid DataSource in a new PVC spec\n* Implementation of the PVC DataSource in the CSI external provisioner\n\nCurrently the only feature gate related to DataSources is the VolumeSnapshotDataSource gate.  This KEP would require an additional Data Source related feature gate `VolumeDataSource`.  Going forward we may continue to add additional feature gates for new DataSource types.  This KEP proposes that feature for Alpha, then following through the standard process for graduation based on feedback and stability during it's alpha release cycle.\n\nGiven that the only implementation changes in Kuberenetes is to enable the feature in the API (all of the actual Clone implementation is handled by the CSI Plugin and back end device) the main criteria for completion will be successful implementation and agreement from the CSI community regarding the Kubernetes API.\n\n## Implementation History\n\n1.15 - Alpha status\n\n## Drawbacks [optional]\n\n## Alternatives [optional]\n\nSnapshots and Clones are very closely related, in fact some back ends my implement cloning via snapshots (take a snapshot, create a volume from that snapshot).  Plugins that provide true `smart clone` functionality are strongly discouraged from using this sort of an implementation, instead they should perform an actual clone if they have the ability to do so.\n\nUsers can do this currently with Kubernetes, and it's good, however some back ends have specific clone functionality that is much more efficient, and even for those that don't, this proposal provides a simple one-step process for a user to request a Clone of a volume.  It's also important to note that using this work around requires management of two object for the user, and in some cases those two object are linked and the new volume isn't truly an independent entity.\n\n## Infrastructure Needed [optional]\n\n"
+  },
+  {
+    "id": "14c2878d732784792b2c31626fdfcad2",
+    "title": "ExecutionHook",
+    "authors": ["@jingxu97", "@xing-yang"],
+    "owningSig": "sig-storage",
+    "participatingSigs": ["sig-storage", "sig-node", "sig-apps", "sig-architecture"],
+    "reviewers": ["@saad-ali", "@thockin", "@liyinan926"],
+    "approvers": ["@thockin", "@saad-ali", "@liyinan926"],
+    "editor": "TBD",
+    "creationDate": "2019-01-20",
+    "lastUpdated": "2019-04-25",
+    "status": "implementable",
+    "seeAlso": ["n/a"],
+    "replaces": ["n/a"],
+    "supersededBy": ["n/a"],
+    "markdown": "\n# Title\n\nExecutionHook API Design\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Workflow](#workflow)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n  - [User Stories](#user-stories)\n- [Workarounds](#workarounds)\n- [Alternatives](#alternatives)\n  - [Alternative Option 1a](#alternative-option-1a)\n  - [Alternative Option 1b](#alternative-option-1b)\n  - [Controller Handlings for Option 1a and 1b](#controller-handlings-for-option-1a-and-1b)\n  - [Alternative Option 2](#alternative-option-2)\n  - [Risks and Mitigations](#risks-and-mitigations-1)\n- [Graduation Criteria](#graduation-criteria-1)\n- [Implementation History](#implementation-history-1)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis proposal is to introduce an API (ExecutionHook) for dynamically executing user’s commands in a pod/container or a group of pods/containers and a controller (ExecutionHookController) to manage the hook lifecycle. ExecutionHook provides a general mechanism for users to trigger hook commands in their containers for their different use cases. Different options have been evaluated to decide how this ExecutionHook should be managed and executed. The preferred option is described in the Proposal section. The other options are discussed in the Alternatives section.\n\n## Motivation\n\nThe volume snapshot feature allows creating/deleting volume snapshots, and the ability to create new volumes from a snapshot natively using the Kubernetes API. However, application consistency is not guaranteed. An user has to figure out how\nto quiece an application before taking a snapshot and unquiece it after taking the snapshot.\n\nSo we want to introduce an `ExecutionHook` to facilitate the quiesce and unquiesce actions when taking a snapshot. There is an existing lifecycle hook in the `Container` struct. The lifecycle hook is called immediately after a container is created or immediately before a container is terminated. The proposed execution hook is not tied to the start or termination time of the container. It can be triggered on demand by callers (users or controllers) and the status will be updated dynamically.\n\n### Goals\n\n`ExecutionHook` is introduced to define actions that can be taken on a container or a group of containers. The ExecutionHook controller is responsible for triggering the commands in containers and updating the status on whether the execution is succeeded or not. The controller will also garbage collect the hook objects based on some predefined policy.\n\n### Non-Goals\n\nThis proposal does not provide exact command included in the `ExecutionHook`\nbecause every application has a different requirement.\n\nApplicationSnapshot and GroupSnapshot will be mentioned in this proposal whenever relevant, but detailed design will not be included in this spec.\n\n## Proposal\n\nThe general ExecutionHook API has two parts, spec and status. The hook spec has two piece of information, what are the commands to execute and where to execute them. In many use cases, different hooks share the same commands and user or controller will create many hooks for execution repeatedly. To reduce the work of copying the same execution commands in different hooks, we also propose a second API, HookAction to record the execution commands which can be referenced by ExecutionHook API. Both APIs are namespaced and they should be in the same namespace.\n\nThe Create event of ExecutionHook will trigger the hook controller to run HookAction command. The status will be updated once it is available by controller. It is the caller's (user or controller) responsibility to delete the hook once the execution finishes (either succeeded or failed). Otherwise, ExecutionHook controller will garbage collect them after a certain amount of time.\n\nHere is the definition of an ExecutionHook API object:\n\n```\n// ExecutionHook is in the tenant namespace\ntype ExecutionHook struct {\n        metav1.TypeMeta\n        // +optional\n        metav1.ObjectMeta\n\n        // Spec defines the behavior of a hook.\n        // +optional\n        Spec ExecutionHookSpec\n\n        // Status defines the current state of a hook.\n        // +optional\n        Status ExecutionHookStatus\n}\n```\n\nHere is the definition of the ExecutionHookSpec:\n\n```\n// HookActionName is copied to ExecutionHookSpec by the controller such as\n// the Snapshot Controller.\ntype ExecutionHookSpec struct {\n        // PodSelection defines how to select pods and containers to run\n\t// the executionhook. If multiple pod/containers are selected, the action will executed on them\n\t// asynchronously. If execution ordering is required, caller has to implement the logic and create\n\t// different hooks in order.\n\t// This field is required.\n        PodSelection PodSelection\n\n        // Name of the HookAction. This is required.\n        ActionName string\n}\n\n// PodSelection contains two fields, PodContainerNamesList and PodContainerSelector,\n// where one of them must be defined so that the hook controller knows where to\n// run the hook.\ntype PodSelection struct {\n        // PodContainerNamesList lists the pods/containers on which the ExecutionHook\n        // should be executed. If not specified, the ExecutionHook controller will find\n        // all pods and containers based on PodContainerSelector.\n        // If both PodContainerNamesList and PodContainerSelector are not\n        // specified, the ExecutionHook cannot be executed and it will fail.\n        // +optional\n        PodContainerNamesList []PodContainerNames\n\n        // PodContainerSelector is for hook controller to find pods and containers\n        // based on the pod label selector and container names\n        // If PodContainerNamesList is specified, this field will not be used.\n        // +optional\n        PodContainerSelector *PodContainerSelector\n}\n\ntype PodContainerNames struct {\n        PodName string\n        ContainerNames []string\n}\n\ntype PodContainerSelector struct {\n\t// PodSelector specifies a label query over a set of pods.\n        // +optional\n\tPodSelector *metav1.LabelSelector\n\n        // If specified, controller only select the containers that are listed from the selected pods based on PodSelector. \n\t// Otherwise, all containers of the pods will be selected\n        // +optional\n        ContainerList []string\n }\n```\n\nHere is the definition of the ExecutionHookStatus. This represents the current state of a hook for all selected containers in the selected pods.\n\n```\n// ExecutionHookStatus represents the current state of a hook\ntype ExecutionHookStatus struct {\n        // This is a list of ContainerExecutionHookStatus, with each status representing information\n        // about how hook is executed in a container, including pod name, container name, ActionTimestamp,\n        // ActionSucceed, etc.\n        // +optional\n        HookStatuses []ContainerExecutionHookStatus\n}\n\n// ContainerExecutionHookStatus represents the current state of a hook for a specific container in a pod\ntype ContainerExecutionHookStatus struct {\n        // This field is required\n        PodName string\n\n        // This field is required\n        ContainerName string\n\n        // If not set, it is nil, indicating Action has not started\n        // If set, it means Action has started at the specified time\n\t// +optional\n        Timestamp *int64\n\n        // ActionSucceed is set to true when the action is executed in the container successfully.\n\t// It will be set to false if the action cannot be executed successfully after ActionTimeoutSeconds passes.\n        // +optional\n        Succeed *bool\n\n        // The last error encountered when executing the action. The hook controller might update this field each time\n\t// it retries the execution.\n\t// +optional\n        Error *HookError\n}\n\ntype HookError struct {\n        // Type of the error\n        // This is required\n        ErrorType ErrorType\n\n        // Error message\n        // +optional\n        Message *string\n\n        // More detailed reason why error happens\n        // +optional\n        Reason *string\n\t\n        // It indicates when the error occurred\n\t// +optional\n        Timestamp *int64\n}\n\ntype ErrorType string\n\n// More error types could be added, e.g., Forbidden, Unauthorized, AlreadyInProgress, etc.\nconst (\n        // The execution hook times out\n        Timeout ErrorType = \"Timeout\"\n\n        // The execution hook fails with an error\n        Error ErrorType = \"Error\"\n)\n```\n\nIn the ExecutionHookStatus object, there is a list of ContainerExecutionHookStatus for all selected containers in the pods, each ContainerExecutionHookStatus represents the state of the hook on a specific container.\n\nHere is the definition of HookAction:\n\n```\n// HookAction describes action commands to run on pods/containers based\n// on specified policies. HookAction will be created by the user and\n// can be re-used later. Snapshot Controller will create ExecutionHooks\n// based on HookActions specified in the snapshot spec. For example,\n// two HookActions, preSnapshotExecutionHook and postSnapshotExecutionHook,\n// are expected in the snapshot spec.\n// HookAction does not contain information on pods/containers because those are\n// runtime info.\n// HookAction is namespaced\ntype HookAction struct {\n        metav1.TypeMeta\n        // +optional\n        metav1.ObjectMeta\n\n        // This contains the command to run on a container.\n\t// The command should be idempotent because the system does not guarantee exactly-once semantics.\n\t// Any action may be triggered more than once but only the latest results will be logged in status.\n\t// As alpha feature, only ExecAction type in Handler will be support, not the HTTPGETAction or TCPSocketAction.\n        // This is required.\n        Action core_v1.Handler\n\n        // ActionTimeoutSeconds defines when the execution hook controller should stop retrying.\n        // If execution fails, the execution hook controller will keep retrying until reaching\n        // ActionTimeoutSeconds. If execution still fails or hangs, execution hook controller\n        // stops retrying and updates executionhook status to failed.\n        // If controller loses its state, counter restarts. In this case, controller will retry\n        // for at least this long, before stopping.\n        // Once an action is started, controller has no way to stop it even if\n        // ActionTimeoutSeconds is exceeded. This simply controls if retry happens or not.\n        // retry is based on exponential backoff policy. If ActionTimeoutSeconds is not\n        // specified, it will retry until the hook object is deleted.\n        // +optional\n        ActionTimeoutSeconds *int64\n}\n```\n\n`ExecutionHook` may be used for different cases, such as\n\n* Application-consistency snapshotting\n* Upgrade\n* Rolling upgrade\n* Debugging\n* Prepare for some lifecycle event like a database migration\n* Reload a config file\n* Restart a container\n\n\nThe following gives an example of how to use ExecutionHook for application-consistency snapshotting use case.\n\n### Workflow\n\n* Snapshot API should carry two execution hook information (which will reference two HookAction - preSnapshotExecutionHook and postSnapshotExecutionHook) for quiescing and unquiescing application.\n* Snapshot controller watches Snapshot objects. If there is a request to create a Snapshot, it checks if there is ExecutionHook Information defined in Snapshot.\n* If hook is defined in the snapshot, snapshot controller will create one or multiple ExecutionHooks for quiescing application if necessary, one for each command running in pods/containers.\n* Snapshot controller waits for all Action hooks to complete running before taking snapshot.\n* The ExecutionHook controller watches the ExecutionHook API object and take actions based on the object status and also update the status. \n* Snapshot controller waits until hook is run on all pods (if more than 1 pod). No matter the Action succeeds or fails, snapshot controller should create execution hooks for unquiescing application. Snapshot controller can also decide when to delete those hooks.\n\nHere is an example of an HookAction:\n```\napiVersion: apps.k8s.io/v1alpha1\nkind: HookAction\nmetadata:\n  name: action-demo\nAction:\n  exec:\n    command: [“run_quiesce.sh”]\n  actionTimeoutSeconds: 10\n```\n\nHere is an ExecutionHook created by the snapshot controller:\n```\napiVersion: apps.k8s.io/v1alpha1\nkind: ExecutionHook\nmetadata:\n  name: hook-demo\nspec:\n  podContainerNamesList:\n    -podName: myPod1\n      -containerName: myContainer1\n      -containerName: myContainer2\n    -podName: myPod2\n      -containerName: myContainer3\n      -containerName: myContainer4\n  hookActionName: action-demo\n```\n\nThe following RBAC rules will be added for the ExecutionHook controller to run the hook through the pod subresource exec:\n```\n  - apiGroups: [\"\"]\n    resources: [\"pods\"]\n    verbs: [\"get\", \"list\", \"watch\"]\n  - apiGroups: [\"\"]\n    resources: [\"pods/exec\"]\n    verbs: [\"create\"]\n```\n\n### Risks and Mitigations\nThe security concern is that ExecutionHook controller has the authority to execute commands in any pods. For alpha and proof of concept, we propose to use external controller to handle executionhooks. But to move to beta and graduate as GA, we will evaluate it and move it to kubelet which already has the privilege to execute commands in pod/containers.\n\n## Graduation Criteria\nPlease see above Risks and Mitigations\n\n## Implementation History\n\n* Feature description: https://github.com/kubernetes/enhancements/issues/962\n\n\n### User Stories\n\n## Workarounds\n\n## Alternatives\n\nThe user can use Annotations to define the execution hook if it is not in the\ncontainer struct.\n\nWe also considered several options based on feedback from design meetings and spec reviews.\n\n### Alternative Option 1a\n\nDefine ExecutionHook as a struct, not an API object, and add it to Lifecycle in the Container struct.\n\nAn `ExecutionHook` is defined in the following.\n\n```\n// ExecutionHook defines a specific action that should be taken with timeout\ntype ExecutionHook struct {\n        // Name of an ExecutionHook\n        Name string `json:\"name\" protobuf:\"bytes,1,opt,name=name\"`\n        // Type of an ExecutionHook\n        Type string `json:\"type\" protobuf:\"bytes,2,opt,name=type\"`\n        // Command to execute for a particular trigger\n        Handler `json:\",inline\" protobuf:\"bytes,3,opt,name=handler\"`\n        // How long the controller should wait for the hook to complete execution\n        // before giving up. The controller needs to succeed or fail within this\n        // timeout period, regardless of the retries. If not set, the controller\n        // will set a default timeout.\n        // +optional\n        TimeoutSeconds int64 `json:\"timeoutSeconds,omitempty\" protobuf:\"varint,4,opt,name=timeoutSeconds\"`\n}\n```\n\nAn `ExecutionHook` includes a name and a type. The type has to be one of valid types\nspecified for `ExecutionHookType` as follows.\n\n```\ntype ExecutionHookType string\n\n// These are valid types of ExecutionHooks\nconst (\n        // An ExecutionHook that freezes an application\n        ExecutionHookFreeze ExecutionHookType = \"Freeze\"\n        // An ExecutionHook that thaws an application\n        ExecutionHookThaw ExecutionHookType = \"Thaw\"\n)\n```\n\nAdd `ExecutionHook` to `Lifecycle`.\n\n```\ntype Lifecycle struct {\n        // PostStart is called immediately after a container is created.  If the handler fails, the container\n        // is terminated and restarted.\n        // +optional\n        PostStart *Handler\n        // Defines the hooks to trigger an action in a container\n        // +optional\n        ExecutionHooks []ExecutionHook\n        // PreStop is called immediately before a container is terminated.  The reason for termination is\n        // passed to the handler.  Regardless of the outcome of the handler, the container is eventually\n        // terminated.\n        // +optional\n        PreStop *Handler\n}\n```\n\n### Alternative Option 1b\n\nAdd `Name`, `Type`, and `TimeOutSeconds` from the `ExecutionsHook` struct directly to the Handler struct. In that case we have to make `Type` optional for backward compatibility.\n\n```\ntype Handler struct {\n        // One and only one of the following should be specified.\n        // Exec specifies the action to take.\n        // +optional\n        Exec *ExecAction\n        // HTTPGet specifies the http request to perform.\n        // +optional\n        HTTPGet *HTTPGetAction\n        // TCPSocket specifies an action involving a TCP port.\n        // TODO: implement a realistic TCP lifecycle hook\n        // +optional\n        TCPSocket *TCPSocketAction\n        // Name of an execution hook\n        // +optional\n        Name string\n        // Type of an execution hook\n        // +optional\n        Type string\n        // How long the controller should wait for the hook to complete execution\n        // before giving up. The controller needs to succeed or fail within this\n        // timeout period, regardless of the retries. If not set, the controller\n        // will set a default timeout.\n        // +optional\n        TimeoutSeconds int64\n}\n\ntype Lifecycle struct {\n        // PostStart is called immediately after a container is created.  If the handler fails, the container\n        // is terminated and restarted.\n        // +optional\n        PostStart *Handler\n        // Defines the hooks to trigger an action in a container\n        // +optional\n        ExecutionHooks []Handler\n        // PreStop is called immediately before a container is terminated.  The reason for termination is\n        // passed to the handler.  Regardless of the outcome of the handler, the container is eventually\n        // terminated.\n        // +optional\n        PreStop *Handler\n}\n```\n\n### Controller Handlings for Option 1a and 1b\n\nThere are two options about which component takes care of these hooks.\n* Kubelet\n  Kubelet is responsible for handling the lifecycle hooks during starting/stopping containers. So kubelet could be the central place to handle other types of generic execution hooks. However, there are a number of challenges to support it now.\n  * It is not clear when and how kubelet needs to trigger these actions, how to report the status of executing these hooks, whether it fails or succeeds.\n  * If kubelet supports the execution hook, it should be designed to support general use cases instead of only specific use cases. However, at this stage, we don’t have enough information to determine how it should be defined generally to support a variety of use cases.\n  * It probably takes more time to enable kubelet to support this considering the complexity and all edge cases.\n\n* External Controller\n  Even though the API is defined inside of the container spec, it is possible to allow external controller to use Pod.exec subresource to run arbitrary commands when needed. For example, snapshot controller can trigger freeze hook before taking the snapshot. It is easier for controller to understand the failure and handle the workflow of snapshotting. However, the big concern about this approach is to allow external controllers to execute untrusted code. It would be easy to lose control of how the hooks can be used without careful security considerations.\n\nHere is an example of how the ExecutionHook is used in a container:\n\n```\napiVersion: v1\nkind: Pod\nmetadata:\n  name: hook-demo\nspec:\n  containers:\n  - name: demo-container\n    image: mysql\n    executionHooks:\n      - name: freeze\n        type: Freeze\n        exec:\n          command: [“run_quiesce.sh”]\n        timeoutSeconds: 30\n      - name: thaw\n        type: Thaw\n        exec:\n          command: [“run_unquiesce.sh”]\n        timeoutSeconds: 30\n```\n\nThe following RBAC rules will be added for the snapshot controller if the snapshot controller is responsible for executing the hook:\n\n```\n- apiGroups: [\"\"]\n    resources: [\"pods\"]\n    verbs: [\"get\", \"list\", \"watch\"]\n  - apiGroups: [\"\"]\n    resources: [\"pods/exec\"]\n    verbs: [\"create\"]\n```\n\nPros for option 1 and 2: All the handlers are together in the `Lifecycle` struct. It looks clean.\n\nCons for option 1: `ExecutionHook` has its own types and looks different from the other handlers.\n\nCons for option 1 and 2:\n\n* `Type` is optional here for backward compatibility, however, `Type` should be required because a `Freeze` type tells the controller it needs to be run before cutting the snapshot while a `Thaw` type tells the controller to run it after cutting the snapshot. Making it optional will make it hard for the snapshot controller to determine when to run those hooks.\n* User may get confused. User may expect everything inside `Lifecycle` is handled by kubelet, however, `ExecutionHooks` will be handled by an external controller/operator.\n\nOther differences between `ExecutionHook` and existing `Lifecycle` struct:\n\n* The `ExecutionHook` is triggered by an external controller/operator using pod subresource exec.  Hook can be triggered at any point in the lifecycle of the container based on the `Type`.\n* `Lifecycle` is triggered and handled by kubelet in the beginning and end of the container lifecycle.\n\n### Alternative Option 2\n\nA second option is to add this hook information into the VolumeSnapshot spec. The hook struct needs to specify where these hooks should be applied to, normally through a pod selector. Or, we can use a new CRD to store this information. The controller which requests taking snapshots will trigger these hook to specified Pods/containers. The concern is similar to the above mentioned to allow different external controllers to execute these arbitrary user defined hooks.\n\n```\napiVersion: storage.org/v1alpha1\nkind: PrePostSnapshotHook\nmetadata:\n  name: prepostsnap-rule\nspec:\n  - podSelector:\n      app: mysql\n    actions:\n    - type: command\n      # this command will flush tables with read lock\n      value: mysql --user=root --password=$MYSQL_ROOT_PASSWORD -Bse 'flush tables with read lock'\n```\n\n### Risks and Mitigations\n\n## Graduation Criteria\n\nWhen the existing volume snapshot alpha feature goes beta, the `ExecutionHook`\nfeature will become beta as well.\n\n## Implementation History\n\n* Feature description: https://github.com/kubernetes/enhancements/issues/962\n"
+  },
+  {
+    "id": "31ae46a2c52c2469921f0d0dba55bcc7",
+    "title": "Ephemeral Inline CSI Volumes",
+    "authors": ["@vladimirvivien", "@pohly"],
+    "owningSig": "sig-storage",
+    "participatingSigs": ["sig-storage"],
+    "reviewers": ["@msau42", "@jsafrane", "@liggitt"],
+    "approvers": ["@thockin", "@saad-ali"],
+    "editor": "",
+    "creationDate": "2019-01-22",
+    "lastUpdated": "2019-08-30",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Ephemeral Inline CSI volumes\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-goals](#non-goals)\n- [User stories](#user-stories)\n  - [Examples](#examples)\n- [Ephemeral inline volume proposal](#ephemeral-inline-volume-proposal)\n  - [VolumeHandle generation](#volumehandle-generation)\n  - [API updates](#api-updates)\n  - [Support for inline CSI volumes](#support-for-inline-csi-volumes)\n  - [Secret reference](#secret-reference)\n  - [Specifying allowed inline drivers with \u003ccode\u003ePodSecurityPolicy\u003c/code\u003e](#specifying-allowed-inline-drivers-with-)\n  - [Ephemeral inline volume operations](#ephemeral-inline-volume-operations)\n- [Test plans](#test-plans)\n  - [All unit tests](#all-unit-tests)\n  - [Ephemeral inline volumes unit tests](#ephemeral-inline-volumes-unit-tests)\n  - [E2E tests](#e2e-tests)\n- [Alternatives](#alternatives)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\nCurrently, volumes that are backed by CSI drivers can only be used with the `PersistentVolume` and `PersistentVolumeClaim` objects. This proposal is to implement support for the ability to nest CSI volume declarations within pod specs for ephemeral-style drivers.\n\nThis KEP started life as [feature #2273](https://github.com/kubernetes/community/pull/2273).  Please follow that link for historical context.\n\n\n## Motivation\nImplementing support for embedding volumes directly in pod specs would allow driver developers to create new types of CSI drivers such as ephemeral volume drivers.  They can be used to inject arbitrary states, such as configuration, secrets, identity, variables or similar information, directly inside pods using a mounted volume. \n\n\n### Goals \n* Provide a high level design for ephemeral inline CSI volumes support\n* Define API changes needed to support this feature\n* Outlines how ephemeral inline CSI volumes would work \n* Ensure that inline CSI volumes usage is secure\n\n### Non-goals\nThe followings will not be addressed by this KEP:\n* Introduce new CSI spec changes to support this feature\n* Introduce required changes to existing CSI drivers for this feature\n* Support for topology or pod placement scheme for ephemeral inline volumes\n* Support for PV/PVC related features such as topology, raw block, mount options, and resizing\n* Support for inline pod specs backed by a persistent volumes\n\n## User stories\n* As a storage provider, I want to use the CSI API to develop drivers that can mount ephemeral volumes that follow the lifecycles of pods where they are embedded.   This feature would allow me to create drivers that work similarly to how the in-tree Secrets or ConfigMaps driver works.  My ephemeral CSI driver should allow me to inject arbitrary data into a pod using a volume mount point inside the pod. \n* As a user I want to be able to define pod specs with embedded ephemeral CSI volumes that are created/mounted when the pod is deployed and are deleted when the pod goes away.\n\n### Examples\n\nA pod spec with an ephemeral inline CSI volume.  Note that because the volume is expected to be ephemeral, the `volumeHandle` is not provided.  Instead a CSI-generated ID will be submitted to the driver.\n\n```yaml\napiVersion: v1\nkind: Pod\nmetadata:\n  name: some-pod\nspec:\n  containers:\n    ...\n  volumes:\n      - name: vol\n        csi:\n          driver: some-csi-driver.example.com\n          # Passed as NodePublishVolumeRequest.volume_context,\n          # valid options depend on the driver.\n          volumeAttributes:\n              foo: bar\n```\n\n## Ephemeral inline volume proposal\nA CSI driver may be able to support either PV/PVC-originated or pod spec originated volumes. When a volume definition is embedded inside a pod spec, it is considered to be an `ephemeral inline` volume request and can only participate in *mount/unmount* volume operation calls.  Ephemeral inline volume requests have the following characteristics: \n* The inline volume spec will not contain nor require a `volumeHandle`.\n* The CSI Kubelet plugin will internally generate a `volumeHandle` which is passed to the driver.\n* Using existing strategy, the volumeHandle will be cached for future volume operations (i.e. unmount).\n* The Kubelet will send mount related calls to CSI drivers:\n  * Kubelet will have access to both podUID and pod namespace during mount/Setup operations.\n  * Secrets references can be fully realized during mount/Setup phase and sent to driver.\n* The Kubelet will send unmount related calls to CSI drivers:\n  * The cached volumeHandle will be sent to the driver during unmount/Teardown phase.\n\n### VolumeHandle generation\nDuring mount operation, the Kubelet (internal CSI code) will employ a naming strategy to generate the value for the `volumeHandle`.  The generated value will be a combination of `podUID` and `pod.spec.Volume[x].name` to guarantee uniqueness.  The generated value will be stable and the Kubelet will be able to regenerate the value, if needed, during different phases of storage operations.\n\nThis approach provides several advantages:\n* It makes sure that each pod can use a different volume handle ID for its ephemeral volumes.  \n* Each pod will get a uniquely generated volume handle, preventing accidental naming conflicts in pods.\n* Each pod created by ReplicaSet, StatefulSet or DaemonSet will get the same copy of a pod template. This makes sure that each pod gets its own unique volume handle ID and thus can get its own volume instance.\n\nWithout an auto-generated naming strategy for the `volumeHandle` during an ephemeral lifecycle, a user could guess the volume handle ID of another user causing a security risk. Having a strategy that generates consistent volume handle names, will ensure that drivers obeying idempotency will always return the same volume associated with the podUID. \n\n### API updates\n\nThere are couple of objects needed to implement this feature:\n* `VolumeSource` - object that represents a pod's volume. It will be modified to include CSI volume source.\n* `CSIVolumeSource` - a new object representing the inline volume data coming from the pod.\n\n```go\ntype VolumeSource struct {\n    // \u003csnip\u003e\n    // CSI (Container Storage Interface) represents storage that handled by an external CSI driver (Beta feature).\n    // +optional\n    CSI *CSIVolumeSource\n}\n\n// Represents a source location of a volume to mount, managed by an external CSI driver\ntype CSIVolumeSource struct {\n\t// Driver is the name of the driver to use for this volume.\n\t// Required.\n\tDriver string\n\n\t// Optional: The value to pass to ControllerPublishVolumeRequest.\n\t// Defaults to false (read/write).\n\t// +optional\n\tReadOnly *bool\n\n\t// Filesystem type to mount. Ex. \"ext4\", \"xfs\", \"ntfs\".\n\t// If not provided, the empty value is passed to the associated CSI driver\n\t// which will determine the default filesystem to apply.\n\t// +optional\n\tFSType *string\n\n\t// VolumeAttributes store immutable properties of the volume copied during provision.\n\t// These attributes are passed back to the driver during controller publish calls.\n\t// +optional\n\tVolumeAttributes map[string]string\n\n\t// NodePublishSecretRef is a reference to the secret object containing\n\t// sensitive information to pass to the CSI driver to complete the CSI\n\t// NodePublishVolume and NodeUnpublishVolume calls.\n\t// This field is optional, and  may be empty if no secret is required. If the\n\t// secret object contains more than one secret, all secret references are passed.\n\t// +optional\n\tNodePublishSecretRef *LocalObjectReference\n}\n```\n\n### Support for inline CSI volumes\n\nTo indicate that the driver will support ephemeral inline volume requests, the existing `CSIDriver` object will be extended to include attribute `VolumeLifecycleModes`,\na list of strings. That list may contain:\n- `persistent` if the driver supports normal, persistent volumes (i.e. the normal CSI API); this is the default if nothing is specified\n- `ephemeral` if the driver supports inline CSI volumes\n\nKubelet will check for support for ephemeral volumes before invoking\nthe CSI driver as described next. This prevents accidentally using a\nCSI driver in a way which it doesn't support. This is important\nbecause using a driver incorrectly might end up causing data loss or\nother problems.\n\nWhen a CSI driver supports it, the following approach is used:\n* Volume requests will originate from pod specs.\n* The driver will only receive volume operation calls during mount/unmount phase (`NodePublishVolume`, `NodeUnpublishVolume`)\n* The driver will not receive separate gRPC calls for provisioning, attaching, detaching, and deleting of volumes.\n* The driver is responsible for implementing steps to ensure the volume is created and made available to the pod during mount call.\n* The Kubelet may attempt to mount a path, with the same generated volumeHandle, more than once. If that happens, the driver should be able to handle such cases gracefully.\n* The driver is responsible for implementing steps to delete and clean up any volume and resources during the unmount call.\n* The Kubelet may attempt to call unmount, with the same generated volumeHandle, more than once. If that happens, the driver should be able to handle such cases gracefully.\n* `CSIVolumeSource.FSType` is mapped to `NodePublishVolumeRequest.access_type.mount.fs_type`.\n* All other parameters that a driver might need (like volume size)\n  have to be specified in `CSIVolumeSource.VolumeAttributes` and will be passed in\n  `NodePublishVolumeRequest.volume_context`. What those parameters are is entirely\n  up to the CSI driver.\n\nA driver that supports both modes may need to distinguish in\n`NodePublishVolume` whether the volume is ephemeral or persistent.\nThis can be done by enabling the \"[pod info on\nmount](https://kubernetes-csi.github.io/docs/csi-driver-object.html#what-fields-does-the-csidriver-object-have)\"\nfeature which then, in addition to information about the pod, will\nalso set an entry with this key in the `NodePublishRequest.volume_context`:\n* `csi.storage.k8s.io/ephemeral`: `true` for ephemeral inline volumes, `false` otherwise\n\n### Secret reference\nThe secret reference declared in an ephemeral inline volume can only be used with namespaces from pods where it is referenced.  The `NodePublishSecretRef` is stored in a `LocalObjectReference` value:\n* `LocalObjectReference` do not include a namespace reference.  This is to prevent reference to arbitrary namespace values.\n* The namespace needed will be extracted from the the pod spec by the Kubelet code during mount.\n\n### Specifying allowed inline drivers with `PodSecurityPolicy`\nTo control which CSI driver is allowed to be use ephemeral inline volumes within a pod spec, a new `PodSecurityPolicy` called `AllowedCSIDrivers` is introduced as shown below:\n\n```go\n  type PodSecurityPolicySpec struct {\n\t// \u003csnip\u003e\n\n\t// AllowedCSIDrivers is a whitelist of allowed CSI drivers used inline in a pod spec.  Empty or nil indicates that no\n\t// CSI drivers may be used in this way. This parameter is effective only when the usage of the CSI plugin\n\t// is allowed in the \"Volumes\" field.\n\t// +optional\n\tAllowedCSIDrivers []AllowedCSIDriver\n  }\n\n  // AllowedCSIDriver represents a single CSI driver that is allowed to be used.\n  type AllowedCSIDriver struct {\n\t// Name of the CSI driver\n\tName string\n  }  \n```\n\nValue `PodSecurityPolicy.AllowedCSIDrivers` must be explicitly set with the names of CSI drivers that are allowed to be embedded within a pod spec.  An empty value means no CSI drivers are allowed to be specified inline inside a pod spec.\n\n### Ephemeral inline volume operations\nInline volume requests can only participate in mount/unmount volume operations. This phase is handled by the Kubelet which is responsible for mounting/unmounting device and/or filesystem mount points inside a pod. At mount time, the internal API will pass the volume information via parameter of `volume.Spec` which will contain a value of either type `v1.CSIVolumeSource` (for volume originated from pod specs) or `v1.CSIPersistentVolume` for volume originating from PV/PVC.  The code will check for the presence of a `v1.CSIVolumeSource` or `v1.CSIPersistentVolume` value.  If a `v1.CSIPersistentVolume` is found, the operation is considered non-ephemeral and follows regular PV/PVC execution flow.  If, however, the internal volume API passes a `v1.CSIVolumeSource`:\n* The Kubelet will create necessary mount point paths\n* Kubelet will auto-generate a volumeHandle based on `podUID` and `pod.spec.volume[x].name` (see above for detail).\n* CSI driver will receive mount-like calls (NodePublish) with generated paths and generated volumeHandle.\n\nSince ephemeral volume requests will participate in only the mount/unmount volume operation phase, CSI drivers are responsible for implementing all necessary operations during that phase (i.e. create, mount, unmount, delete, etc).  For instance, a driver would be responsible for provisioning any new volume resource during `NodePublish` and for tearing down these resources during the `NodeUnpublish` calls.\n\n\n## Test plans\n\n### All unit tests\n* Volume operation that use CSIVolumeSource can only work with proper feature gate enabled\n\n### Ephemeral inline volumes unit tests\n* Ensure required fields are provided: csi.storage.k8s.io/ephemeral (https://github.com/pohly/kubernetes/blob/4bc5d065c919fc239e2c8b40e6a96e409ca011fd/pkg/volume/csi/csi_mounter_test.go#L140-L146)\n* Mount/Unmount should be triggered with CSIVolumeSource: https://github.com/kubernetes/kubernetes/blob/10005d2e1e1425904f8c7bf5615e730fb0fea7c9/pkg/volume/csi/csi_mounter_test.go#L386\n* Expected generated volumeHandle is created properly: https://github.com/kubernetes/kubernetes/blob/10005d2e1e1425904f8c7bf5615e730fb0fea7c9/pkg/volume/csi/csi_plugin_test.go#L177\n* Ensure that CSIDriver.Spec.Mode field is validated properly: https://github.com/kubernetes/kubernetes/pull/80568\n* Ensure volumeHandle conforms to resource naming format: TODO\n* CSIVolumeSource info persists in CSI json file during mount/unmount: TODO\n* Ensure Kubelet skips attach/detach when `CSIDriver.Mode = ephemeral`: TODO\n* Ensure Kubelet skips inline logic when `CSIDriver.Mode = persistent` or `CSIDriver.Mode is empty`: covered by existing tests\n\n### E2E tests\n* Pod spec with an ephemeral inline volume request can be mounted/unmounted: https://github.com/pohly/kubernetes/blob/4bc5d065c919fc239e2c8b40e6a96e409ca011fd/test/e2e/storage/csi_mock_volume.go#L356-L371, https://github.com/pohly/kubernetes/blob/4bc5d065c919fc239e2c8b40e6a96e409ca011fd/test/e2e/storage/testsuites/ephemeral.go#L110-L115\n* Two pods accessing an ephemeral inline volume which has the same attributes in both pods: \"should support two pods which share the same data\" in `ephemeral.go` (upcoming PR)\n* Single pod referencing two distinct inline volume request from the same driver: \"should support multiple inline ephemeral volumes\" in `ephemeral.go` (upcoming PR)\n* CSI Kubelet code invokes driver operations during mount for ephemeral volumes: `checkPodLogs` in `csi_mock_volume.go` (upcoming PR)\n* CSI Kubelet code invokes driver operation during unmount of ephemeral volumes: `checkPodLogs` in `csi_mock_volume.go` (upcoming PR)\n* CSI Kubelet cleans up ephemeral volume paths once pod goes away: TODO\n* Apply PodSecurity settings for allowed CSI drivers: TODO\n* Enable testing of an external ephemeral CSI driver: https://github.com/kubernetes/kubernetes/pull/79983/files#diff-e5fc8d9911130b421b74b1ebc273f458\n* Enable testing of the csi-host-path-driver in ephemeral mode in Kubernetes-CSI Prow jobs and Kubernetes itself: TODO\n\n## Alternatives\n\nInstead of allowing CSI drivers that support both ephemeral and\npersistent volumes and passing the `csi.storage.k8s.io/ephemeral`\nhint, a simpler solution would have been to require that a driver gets\ndeployed twice, once for for each kind of volume. That was rejected\nbecause a driver might manage some resource that is shared between\nboth kinds of volumes, like local disks (LVM) or persistent memory\n(PMEM). Having to deploy the driver twice would have made the driver\nimplementation more complex.\n\n## Implementation History\n\n1.15:\n- Alpha status\n- `CSIDriver.Mode` not added yet\n- a CSI driver deployment can only be used for ephemeral inline\n  volumes or persistent volumes, but not both, because the driver\n  cannot determine the mode in its `NodePublishVolume` implementation\n\n1.16:\n- Beta status\n- the same CSI driver deployment can support both modes by enabling\n  the pod info feature and checking the value of\n  `csi.storage.k8s.io/ephemeral`\n  (https://github.com/kubernetes/kubernetes/pull/79983, merged)\n- `CSIDriver.VolumeLifecycleMode` added and checked before calling a CSI driver for\n  an ephemeral inline volume\n  (https://github.com/kubernetes/kubernetes/pull/80568, merged)\n"
+  },
+  {
+    "id": "3b914b43ce3048bc1ae6a38cf7ba2be2",
+    "title": "CSI Volume Topology",
+    "authors": ["@verult"],
+    "owningSig": "sig-storage",
+    "participatingSigs": ["sig-storage"],
+    "reviewers": ["@msau42", "@saad-ali"],
+    "approvers": ["@msau42", "@saad-ali"],
+    "editor": "TBD",
+    "creationDate": "2019-01-24",
+    "lastUpdated": "2019-10-14",
+    "status": "implementable",
+    "seeAlso": ["n/a"],
+    "replaces": ["n/a"],
+    "supersededBy": ["n/a"],
+    "markdown": "\n# Title\n\nCSI Volume Topology\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Upgrade/Downgrade Strategy](#upgradedowngrade-strategy)\n  - [Deprecations](#deprecations)\n- [Version Skew Strategy](#version-skew-strategy)\n- [Test Plan](#test-plan)\n  - [GA testing](#ga-testing)\n- [Graduation Criteria](#graduation-criteria)\n  - [Alpha-\u0026gt;Beta](#alpha-beta)\n  - [Beta-\u0026gt;GA](#beta-ga)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis KEP is written after the original design doc has been approved and implemented. Design for CSI Volume Topology Support in Kubernetes is incorporated as part of the [CSI Volume Plugins in Kubernetes Design Doc](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/container-storage-interface.md).\n\nThe rest of the document includes required information missing from the original design document: test plan and graduation criteria.\n\n## Upgrade/Downgrade Strategy\nWhen moving to GA in K8s 1.17, the storage/v1 CSINode object will be introduced. 1.17\nkubelets will immediately start creating v1 CSINode objects. 1.17 in-tree\ncontrollers (scheduler and attach-detach controller) can switch to using the v1 APIs.\n\nExternal controllers like the csi-provisioner and csi-attacher can start using\nthe v1 objects if they are able to coordinate and align their deployment with the\ndeployment of the API server. However, this may not always be possible, and\nnewer sidecars should be able to continue to work on older Kubernetes versions.\nTherefore, these sidecars need to continue to support both v1beta1 and v1 versions\nfor the duration of the deprecation period. They can try the v1 API first, and then\nfallback to v1beta if the v1 API doesn't exist.\n\n### Deprecations\nThe v1beta1.CSINode object will be deprecated in K8s 1.17, and can be removed in\n1.20 according to the Kubernetes deprecation policy.\n\nWhat that means is in 1.20, CSI drivers using older versions of CSI sidecars\nthat are not aware of v1 objects will stop functioning unless they upgrade to\nnewer versions of the sidecars that are v1-aware.\n\nSimilarly, we will deprecate v1beta1.CSINode support in the CSI sidecars in the\nsidecar release where we introduce v1 support, and remove v1beta1 support from\nthe sidecar in the same release corresponding to K8s 1.20. That removal will\nrequire a new major version of the CSI sidecars.\n\n## Version Skew Strategy\nCSI sidecars will support the scenario with K8s 1.13 nodes where\nCSINode may not have existed until we remove support for v1beta1 in 1.20 and\nbump the major version. This behavior also needs to be deprecated when we\nrelease the v1-aware sidecar version.\n\n## Test Plan\n* Unit tests around topology logic in kubelet and CSI external-provisioner.\n* New e2e tests around topology features will be added in CSI e2e test suites, which test various volume operation behaviors from the perspective of the end user. Tests include:\n  * (Positive) Volume provisioning with immediate volume binding and AllowedTopologies set.\n  * (Positive) Volume provisioning with delayed volume binding.\n  * (Positive) Volume provisioning with delayed volume binding and AllowedTopologies set.\n  * (Negative) Volume provisioning with immediate volume binding and pod zone missing from AllowedTopologies.\n  * (Negative) Volume provisioning with delayed volume binding and pod zone missing from AllowedTopologies.\nInitially topology tests are run against a single CSI driver. As the CSI test suites become modularized they will run against arbitrary CSI drivers.\n\n### GA testing\n* Manual e2e testing for upgrade and version skew scenarios.\n  * An older sidecar that only understands v1beta1 should continue to work when the cluster is\n    upgraded to 1.17. Upgrading the sidecar after cluster upgrade to a version that understands v1 objects should also continue to work.\n  * A newer sidecar that supports v1 and v1beta1 should continue to work if Master and Nodes are \u003c 1.17.\n  * A newer sidecar that supports v1 and v1beta1 should continue to work if Nodes are \u003c 1.17.\n  * A newer sidecar that only supports v1 should work if Master is \u003e= 1.17. Nodes\n    can be \u003e= 1.15 and still use the v1beta object.\n\n## Graduation Criteria\n\n### Alpha-\u003eBeta\n\n* Feature complete, including:\n  * Volume provisioning with required topology constraints\n  * Volume provisioning with preferred topology\n  * Cluster-wide topology aggregation\n  * StatefulSet volume spreading\n* Depends on: CSINodeInfo beta or above; Kubelet Plugins Watcher beta or above\n* Unit and e2e tests implemented\n\n### Beta-\u003eGA\n\n* Depends on: CSINodeInfo GA; Kubelet Plugins Watcher GA\n* Stress test: provisioning load tests; node scale tests; component crash tests\n* Feature deployed in production and have gone through at least one K8s upgrade.\n* Upgrade and version skew testing.\n\n## Implementation History\n\n* K8s 1.12: Alpha\n* K8s 1.14: Beta\n* K8s 1.17: Targeting GA\n"
+  },
+  {
+    "id": "714154229e9fe79c4b442cee27e724d2",
+    "title": "Local Persistent Volumes",
+    "authors": ["msau42", "vishh", "dhirajh", "ianchakeres"],
+    "owningSig": "sig-storage",
+    "participatingSigs": ["sig-storage"],
+    "reviewers": ["saad-ali", "jsafrane", "gnufied"],
+    "approvers": ["saad-ali"],
+    "editor": "TBD",
+    "creationDate": "2019-01-24",
+    "lastUpdated": "2019-01-24",
+    "status": "implementable",
+    "seeAlso": [
+      "https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/volume-topology-scheduling.md"
+    ],
+    "replaces": [
+      "https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/local-storage-pv.md"
+    ],
+    "supersededBy": null,
+    "markdown": "\n# Local Persistent Volumes\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Background](#background)\n  - [Use Cases](#use-cases)\n    - [Distributed filesystems and databases](#distributed-filesystems-and-databases)\n    - [Caching](#caching)\n  - [Environments](#environments)\n    - [Baremetal](#baremetal)\n    - [GCE/GKE](#gcegke)\n    - [EC2](#ec2)\n  - [Limitations of current volumes](#limitations-of-current-volumes)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [PVC Users](#pvc-users)\n    - [Cluster Administrator](#cluster-administrator)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n    - [Local Volume Plugin](#local-volume-plugin)\n      - [API Changes](#api-changes)\n    - [PersistentVolume Node Affinity](#persistentvolume-node-affinity)\n    - [Local volume initial configuration](#local-volume-initial-configuration)\n    - [Local volume management](#local-volume-management)\n      - [Packaging](#packaging)\n    - [Block devices and raw partitions](#block-devices-and-raw-partitions)\n      - [Discovery](#discovery)\n      - [Cleanup after Release](#cleanup-after-release)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Test Plan](#test-plan)\n  - [API unit tests](#api-unit-tests)\n  - [PV node affinity unit tests](#pv-node-affinity-unit-tests)\n  - [Local volume plugin unit tests](#local-volume-plugin-unit-tests)\n  - [Local volume provisioner unit tests](#local-volume-provisioner-unit-tests)\n  - [E2E tests](#e2e-tests)\n  - [Stress tests](#stress-tests)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n  - [K8s 1.7: Alpha](#k8s-17-alpha)\n  - [K8s 1.9: Alpha](#k8s-19-alpha)\n  - [K8s 1.10: Beta](#k8s-110-beta)\n  - [K8s 1.12: Beta](#k8s-112-beta)\n- [Infrastructure Needed](#infrastructure-needed)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis document presents a detailed design for supporting persistent local storage,\nas outlined in [Local Storage Overview](local-storage-overview.md).\n\nThis KEP replaces the original [design\nproposal](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/local-storage-pv.md)\nand has been updated to reflect the current implementation.\n\n\n## Motivation\n\nIn Kubernetes, there are two main types of storage: remote and local.\n\nRemote storage is typically used with persistent volumes where the data can\npersist beyond the lifetime of the pod.\n\nLocal storage is typically used with ephemeral volumes where the data only\npersists during the lifetime of the pod.\n\nThere is increasing demand for using local storage as persistent volumes,\nespecially for distributed filesystems and databases such as GlusterFS and\nCassandra.  The main motivations for using persistent local storage, instead\nof persistent remote storage include:\n\n* Performance:  Local SSDs achieve higher IOPS and throughput than many\nremote storage solutions.\n\n* Cost: Operational costs may be reduced by leveraging existing local storage,\nespecially in bare metal environments.  Network storage can be expensive to\nsetup and maintain, and it may not be necessary for certain applications.\n\n### Goals\n\n* Allow pods to mount any local block or filesystem based volume.\n* Allow pods to mount dedicated local disks, or channeled partitions as volumes for\nIOPS isolation.\n* Allow pods do access local volumes without root privileges.\n* Allow pods to access local volumes without needing to understand the storage\nlayout on every node.\n* Persist local volumes and provide data gravity for pods.  Any pod\nusing the local volume will be scheduled to the same node that the local volume\nis on.\n* Allow pods to specify local storage as part of a Deployment or StatefulSet.\n* Allow administrators to set up and configure local volumes with simple methods.\n* Do not require administrators to manage the local volumes once provisioned\nfor a node.\n\n### Non-Goals\n\n* Node preparation to setup disks for an environment including, but not limited\n  to: partitioning, RAID, and formatting.\n* Allow pods to release their local volume bindings and lose that volume's data\nduring failure conditions, such as node, storage or scheduling failures, where\nthe volume is not accessible for some user-configurable time.\n* Dynamic provisioning of local volumes.\n* Provide data availability for a local volume beyond its local node.\n* Support the use of HostPath volumes and Local PVs on the same volume.\n\n\n## Background\n\n### Use Cases\n\n#### Distributed filesystems and databases\n\nMany distributed filesystem and database implementations, such as Cassandra and\nGlusterFS, utilize the local storage on each node to form a storage cluster.\nThese systems typically have a replication feature that sends copies of the data\nto other nodes in the cluster in order to provide fault tolerance in case of\nnode failures.  Non-distributed, but replicated databases, like MySQL, can also\nutilize local storage to store replicas.\n\nThe main motivations for using local persistent storage are performance and\ncost.  Since the application handles data replication and fault tolerance, these\napplication pods do not need networked storage to provide shared access to data.\nIn addition, installing a high-performing NAS or SAN solution can be more\nexpensive, and more complex to configure and maintain than utilizing local\ndisks, especially if the node was already pre-installed with disks.  Datacenter\ninfrastructure and operational costs can be reduced by increasing storage\nutilization.\n\nThese distributed systems are generally stateful, infrastructure applications\nthat provide data services to higher-level applications.  They are expected to\nrun in a cluster with many other applications potentially sharing the same\nnodes.  Therefore, they expect to have high priority and node resource\nguarantees.  They typically are deployed using StatefulSets, custom\ncontrollers, or operators.\n\n#### Caching\n\nCaching is one of the recommended use cases for ephemeral local storage.  The\ncached data is backed by persistent storage, so local storage data durability is\nnot required.  However, there is a use case for persistent local storage to\nachieve data gravity for large caches.  For large caches, if a pod restarts,\nrebuilding the cache can take a long time.  As an example, rebuilding a 100GB\ncache from a hard disk with 150MB/s read throughput can take around 10 minutes.\nIf the service gets restarted and all the pods have to restart, then performance\nand availability can be impacted while the pods are rebuilding.  If the cache is\npersisted, then cold startup latencies are reduced.\n\nContent-serving applications and producer/consumer workflows commonly utilize\ncaches for better performance.  They are typically deployed using Deployments,\nand could be isolated in its own cluster, or shared with other applications.\n\n### Environments\n\n#### Baremetal\n\nIn a baremetal environment, nodes may be configured with multiple local disks of\nvarying capacity, speeds and mediums.  Mediums include spinning disks (HDDs) and\nsolid-state drives (SSDs), and capacities of each disk can range from hundreds\nof GBs to tens of TB. Multiple disks may be arranged in JBOD or RAID configurations \nto consume as persistent storage.\n\nCurrently, the methods to use the additional disks are to:\n\n* Configure a distributed filesystem\n* Configure a HostPath volume\n\nIt is also possible to configure a NAS or SAN on a node as well.  Speeds and\ncapacities will widely vary depending on the solution.\n\n#### GCE/GKE\n\nGCE and GKE both have a local SSD feature that can create a VM instance with up\nto 8 fixed-size 375GB local SSDs physically attached to the instance host and\nappears as additional disks in the instance.  The local SSDs have to be\nconfigured at the VM creation time and cannot be dynamically attached to an\ninstance later.  If the VM gets shutdown, terminated, pre-empted, or the host\nencounters a non-recoverable error, then the SSD data will be lost.  If the\nguest OS reboots, or a live migration occurs, then the SSD data will be\npreserved.\n\n#### EC2\n\nIn EC2, the instance store feature attaches local HDDs or SSDs to a new instance\nas additional disks.  HDD capacities can go up to 24 2TB disks for the largest\nconfiguration.  SSD capacities can go up to 8 800GB disks or 2 2TB disks for the\nlargest configurations.  Data on the instance store only persists across\ninstance reboot.\n\n### Limitations of current volumes\n\nThe following is an overview of existing volume types in Kubernetes, and how\nthey cannot completely address the use cases for local persistent storage.\n\n* EmptyDir: A temporary directory for a pod that is created under the kubelet\nroot directory.  The contents are deleted when a pod dies.  Limitations:\n\n  * Volume lifetime is bound to the pod lifetime.  Pod failure is more likely\nthan node failure, so there can be increased network and storage activity to\nrecover data via replication and data backups when a replacement pod is started.\n  * Multiple disks are not supported unless the administrator aggregates them\ninto a spanned or RAID volume.  In this case, all the storage is shared, and\nIOPS guarantees cannot be provided.\n  * There is currently no method of distinguishing between HDDs and SDDs.  The\n“medium” field could be expanded, but it is not easily generalizable to\narbitrary types of mediums.\n\n* HostPath: A direct mapping to a specified directory on the node.  The\ndirectory is not managed by the cluster.  Limitations:\n\n  * Admin needs to manually setup directory permissions for the volume’s users.\n  * Admin has to manage the volume lifecycle manually and do cleanup of the data and\ndirectories.\n  * All nodes have to have their local storage provisioned the same way in order to\nuse the same pod template.\n  * There can be path collision issues if multiple pods get scheduled to the same\nnode that want the same path\n  * If node affinity is specified, then the user has to do the pod scheduling\nmanually.\n\n* Provider’s block storage (GCE PD, AWS EBS, etc): A remote disk that can be\nattached to a VM instance.  The disk’s lifetime is independent of the pod’s\nlifetime.  Limitations:\n\n  * Doesn’t meet performance requirements.\n[Performance benchmarks on GCE](https://cloud.google.com/compute/docs/disks/performance)\nshow that local SSD can perform better than SSD persistent disks:\n\n    * 16x read IOPS\n    * 11x write IOPS\n    * 6.5x read throughput\n    * 4.5x write throughput\n\n* Networked filesystems (NFS, GlusterFS, etc): A filesystem reachable over the\nnetwork that can provide shared access to data.  Limitations:\n\n  * Requires more configuration and setup, which adds operational burden and\ncost.\n  * Requires a high performance network to achieve equivalent performance as\nlocal disks, especially when compared to high-performance SSDs.\n\nDue to the current limitations in the existing volume types, a new method for\nproviding persistent local storage should be considered.\n\n## Proposal\n\n### User Stories\n\n#### PVC Users\nA user can create a PVC and get access to a local disk just by specifying the appropriate StorageClass.\n\n#### Cluster Administrator\nA cluster administrator can easily expose local disks as PVs to their end users.\n\n### Implementation Details/Notes/Constraints\n\n#### Local Volume Plugin\n\nA new volume plugin will be introduced to represent logical block partitions and\nfilesystem mounts that are local to a node.  Some examples include whole disks,\ndisk partitions, RAID volumes, LVM volumes, or even directories in a shared\npartition.  Multiple Local volumes can be created on a node, and is\naccessed through a local mount point or path that is bind-mounted into the\ncontainer.  It is only consumable as a PersistentVolumeSource because the PV\ninterface solves the pod spec portability problem and provides the following:\n\n* Abstracts volume implementation details for the pod and expresses volume\nrequirements in terms of general concepts, like capacity and class.  This allows\nfor portable configuration, as the pod is not tied to specific volume instances.\n* Allows volume management to be independent of the pod lifecycle.  The volume can\nsurvive container, pod and node restarts.\n* Allows volume classification by StorageClass.\n* Is uniquely identifiable within a cluster and is managed from a cluster-wide\nview.\n\nThere are major changes in PV and pod semantics when using Local volumes\ncompared to the typical remote storage volumes.\n\n* Since Local volumes are fixed to a node, a pod using that volume has to\nalways be scheduled on that node.\n* Volume availability is tied to the node’s availability.  If the node is\nunavailable, then the volume is also unavailable, which impacts pod\navailability.\n* The volume’s data durability characteristics are determined by the underlying\nstorage system, and cannot be guaranteed by the plugin.  A Local volume\nin one environment can provide data durability, but in another environment may\nonly be ephemeral.  As an example, in the GCE/GKE/AWS cloud environments, the\ndata in directly attached, physical SSDs is immediately deleted when the VM\ninstance terminates or becomes unavailable.\n\nDue to these differences in behaviors, Local volumes are not suitable for\ngeneral purpose use cases, and are only suitable for specific applications that\nneed storage performance and data gravity, and can tolerate data loss or\nunavailability.  Applications need to be aware of, and be able to handle these\ndifferences in data durability and availability.\n\nLocal volumes are similar to HostPath volumes in the following ways:\n\n* Partitions need to be configured by the storage administrator beforehand.\n* Volume is referenced by the path to the partition.\n* Provides the same underlying partition’s support for IOPS isolation.\n* Volume is permanently attached to one node.\n* Volume can be mounted by multiple pods on the same node.\n\nHowever, Local volumes will address these current issues with HostPath\nvolumes:\n\n* Security concerns allowing a pod to access any path in a node.  Local\nvolumes cannot be consumed directly by a pod.  They must be specified as a PV\nsource, so only users with storage provisioning privileges can determine which\npaths on a node are available for consumption.\n* Difficulty in permissions setup.  Local volumes will support fsGroup so\nthat the admins do not need to setup the permissions beforehand, tying that\nparticular volume to a specific user/group.  During the mount, the fsGroup\nsettings will be applied on the path.  However, multiple pods\nusing the same volume should use the same fsGroup.\n* Volume lifecycle is not clearly defined, and the volume has to be manually\ncleaned up by users.  For Local volumes, the PV has a clearly defined\nlifecycle.  Upon PVC deletion, the PV will be released (if it has the Delete\npolicy), and all the contents under the path will be deleted.  In the future,\nadvanced cleanup options, like zeroing can also be specified for a more\ncomprehensive cleanup.\n\n##### API Changes\n\nAll new changes are protected by a new feature gate, `PersistentLocalVolumes`.\n\nA new `LocalVolumeSource` type is added as a `PersistentVolumeSource`.\nThe path can only be a mount point, a directory in a shared filesystem, or a\nblock device.\n\nIf it is a block device, then the filesystem type can be specified as well, and\nKubernetes will format the filesystem on the device.\n\n```\ntype LocalVolumeSource struct {\n    // The full path to the volume on the node\n    // It can be either a directory or block device (disk, partition, ...).\n    Path string\n\n    // Filesystem type to mount.\n    // It applies only when the Path is a block device.\n    // Must be a filesystem type supported by the host operating system.\n    // Ex. \"ext4\", \"xfs\", \"ntfs\". The default value is to auto-select a fileystem if unspecified.\n    // +optional\n    FSType *string\n}\n\ntype PersistentVolumeSource struct {\n    \u003csnip\u003e\n    // Local represents directly-attached storage with node affinity.\n    // +optional\n    Local *LocalVolumeSource\n}\n```\n\nThe relationship between a Local volume and its node will be expressed using\nPersistentVolume node affinity, described in the following section.\n\nUsers request Local volumes using PersistentVolumeClaims in the same manner as any\nother volume type. The PVC will bind to a matching PV with the appropriate capacity,\nAccessMode, and StorageClassName.  Then the user specifies that PVC in their\nPod spec.  There are no special annotations or fields that need to be set in the Pod\nor PVC to distinguish between local and remote storage.  It is abstracted by the\nStorageClass.\n\n#### PersistentVolume Node Affinity\n\nPersistentVolume node affinity is a new concept and is similar to Pod node affinity,\nexcept instead of specifying which nodes a Pod has to be scheduled to, it specifies which nodes\na PersistentVolume can be attached and mounted to, influencing scheduling of Pods that\nuse local volumes.\n\nThe scheduler will use a PV's node affinity to influence where a Pod can be\nscheduled, as well as which PVs can be bound to a PVC, taking into account all\nscheduling constraints on the Pod. For more details on this feature, see the\n[volume topology design\nproposal](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/volume-topology-scheduling.md).\n\nLocal volumes require PV node affinity to be set.\n\n#### Local volume initial configuration\n\nThere are countless ways to configure local storage on a node, with different patterns to\nfollow depending on application requirements and use cases.  Some use cases may require\ndedicated disks; others may only need small partitions and are ok with sharing disks.\nInstead of forcing a partitioning scheme on storage administrators, the Local volume\nis represented by a path, and lets the administrators partition their storage however they\nlike, with a few minimum requirements:\n\n* The paths to the mount points are always consistent, even across reboots or when storage\nis added or removed.\n* The paths are backed by a filesystem\n* The directories have appropriate permissions for the provisioner to be able to set owners and\ncleanup the volume.\n\n#### Local volume management\n\nLocal PVs are statically created and not dynamically provisioned.\nTo mitigate the amount of time an administrator has to spend managing Local volumes,\na Local static provisioner application will be provided to handle common scenarios.  For\nuncommon scenarios, a specialized provisioner can be written.\n\nThe Local static provisioner will be developed in an external repository,\nand will loosely follow the external provisioner design, with a few differences:\n\n* A provisioner instance needs to run on each node and only manage the local storage on its node.\n* It does not handle dynamic provisioning.  Instead, it performs static provisioning\nby discovering available partitions mounted under configurable discovery directories.\n\nThe basic design of the provisioner will have two separate handlers: one for PV deletion and\ncleanup, and the other for static PV creation.  A PersistentVolume informer will be created\nand its cache will be used by both handlers.\n\nPV deletion will operate on the Update event.  If the PV it provisioned changes to the “Released”\nstate, and if the reclaim policy is Delete, then it will cleanup the volume and then delete the PV,\nremoving it from the cache.\n\nPV creation does not operate on any informer events.  Instead, it periodically monitors the discovery\ndirectories, and will create a new PV for each path in the directory that is not in the PV cache.  It\nsets the \"pv.kubernetes.io/provisioned-by\" annotation so that it can distinguish which PVs it created.\n\nThe allowed discovery file types are directories, mount points, and block\ndevices.  The PV capacity\nwill be the capacity of the underlying filesystem.  Therefore, PVs that are backed by shared\ndirectories will report its capacity as the entire filesystem, potentially causing overcommittment.\nSeparate partitions are recommended for capacity isolation.\n\nThe name of the PV needs to be unique across the cluster.  The provisioner will hash the node name,\nStorageClass name, and base file name in the volume path to generate a unique name.\n\n##### Packaging\n\nThe provisioner is packaged as a container image and will run on each node in the cluster as part of\na DaemonSet.  It needs to be run with a user or service account with the following permissions:\n\n* Create/delete/list/get PersistentVolumes - Can use the `system:persistentvolumeprovisioner` ClusterRoleBinding\n* Get ConfigMaps - To access user configuration for the provisioner\n* Get Nodes - To get the node's UID and labels\n\nThese are broader permissions than necessary (a node's access to PVs should be restricted to only\nthose local to the node).  A redesign will be considered in a future release to address this issue.\n\nIn addition, it should run with high priority so that it can reliably handle all the local storage\npartitions on each node, and with enough permissions to be able to cleanup volume contents upon\ndeletion.\n\nThe provisioner DaemonSet requires the following configuration:\n\n* The node's name set as the MY_NODE_NAME environment variable\n* ConfigMap with StorageClass -\u003e discovery directory mappings\n* Each mapping in the ConfigMap needs a hostPath volume\n* User/service account with all the required permissions\n\nHere is an example ConfigMap:\n\n```\nkind: ConfigMap\nmetadata:\n  name: local-volume-config\n  namespace: kube-system\ndata:\n  storageClassMap: |\n    local-fast:\n      hostDir: \"/mnt/ssds\"\n      mountDir: \"/local-ssds\"\n    local-slow:\n      hostDir: \"/mnt/hdds\"\n      mountDir: \"/local-hdds\"\n```\n\nThe `hostDir` is the discovery path on the host, and the `mountDir` is the path it is mounted to in\nthe provisioner container.  The `hostDir` is required because the provisioner needs to create Local PVs\nwith the `Path` based off of `hostDir`, not `mountDir`.\n\nThe DaemonSet for this example looks like:\n```\n\napiVersion: extensions/v1beta1\nkind: DaemonSet\nmetadata:\n  name: local-storage-provisioner\n  namespace: kube-system\nspec:\n  template:\n    metadata:\n      labels:\n        system: local-storage-provisioner\n    spec:\n      containers:\n      - name: provisioner\n        image: \"k8s.gcr.io/local-storage-provisioner:v1.0\"\n        imagePullPolicy: Always\n        volumeMounts:\n        - name: vol1\n          mountPath: \"/local-ssds\"\n        - name: vol2\n          mountPath: \"/local-hdds\"\n        env:\n        - name: MY_NODE_NAME\n          valueFrom:\n            fieldRef:\n              fieldPath: spec.nodeName\n      volumes:\n      - name: vol1\n        hostPath:\n          path: \"/mnt/ssds\"\n      - name: vol2\n        hostPath:\n          path: \"/mnt/hdds\"\n      serviceAccount: local-storage-admin\n```\n\nA Helm chart can be created to help generate the specs.\n\n#### Block devices and raw partitions\n\nPods accessing raw block storage is a new alpha feature in 1.9.  Changes are required in\nthe Local volume plugin and provisioner to be able to support raw block devices. The local\nvolume provisioner will be enhanced to support discovery of block devices and creation of\nPVs corresponding to those block devices. In addition, when a block device based PV is\nreleased, the local volume provisioner will cleanup the block devices. The cleanup\nmechanism will be configurable and also customizable as no single mechanism covers all use\ncases.\n\n##### Discovery\n\nMuch like the current file based PVs, the local volume provisioner will look for block devices\nunder designated directories that have been mounted on the provisioner container. Currently, for\neach storage class, the provisioner has a configmap entry that looks like this:\n\n```\ndata:\n  storageClassMap: |\n    local-fast:\n      hostDir: \"/mnt/disks\"\n      mountDir: \"/local-ssds\"\n```\n\nWith this current approach, filesystems that were meant to be exposed as PVs are supposed to be\nmounted on sub-directories under hostDir and the provisioner running in a container would walk\nthrough the corresponding \"mountDir\" to find all the PVs.  \n\nFor block discovery, we will extend the same approach to enable discovering block devices. The\nadmin can create symbolic links under hostDir for each block device that should be discovered\nunder that storage class. The provisioner would use the same configMap and its logic will be\nenhanced to auto detect if the entry under the directory is a block device or a file system. If\nit is a block device, then a block based PV is created, otherwise a file based PV is created.\n\n##### Cleanup after Release\n\nCleanup of a block device can be a bit more involved for the following reasons:\n\n* With file based PVs, a quick deletion of all files (inode information) was sufficient, with\nblock devices one might want to wipe all current content.\n* Overwriting SSDs is not guaranteed to securely cleanup all previous content as there is a\nlayer of indirection in SSDs called the FTL (flash translation layer) and also wear leveling\ntechniques in SSDs that prevent reliable overwrite of all previous content. \n* SSDs can also suffer from wear if they are repeatedly subjected to zeroing out, so one would\nneed different tools and strategies for HDDs vs SSDs\n* A cleanup process which favors overwriting every block in the disk can take several hours.\n\nFor this reason, the cleanup process has been made configurable and extensible, so that admin\ncan use the most appropriate method for their environment.\n \nBlock device cleanup logic will be encapsulated in separate scripts or binaries. There will be\nseveral scripts that will be made available out of the box, for example:\n\n\n| Cleanup Method | Description | Suitable for Device |\n|dd-zero| Used for zeroing the device repeatedly | HDD |\n|blkdiscard| Discards sectors on the device. This cleanup method may not be supported by all devices.| SSD |\n|fs-reset| A non-secure overwrite of any existing filesystem with mkfs, followed by wipefs to remove the signature of the file system | SSD/HDD |\n|shred|Repeatedly writes random values to the block device. Less effective with wear levelling in SSDs.| HDD |\n| hdparm| Issues [ATA secure erase](https://ata.wiki.kernel.org/index.php/ATA_Secure_Erase) command to erase data on device. See ATA Secure Erase. Please note that the utility has to be supported by the device in question. | SSD/HDD |\n\nThe fs-reset method is a quick and minimal approach as it does a reset of any file system, which\nworks for both SSD and HDD and will be the default choice for cleaning. For SSDs, admins could\nopt for either blkdiscard which is also quite fast or hdparm. For HDDs they could opt for\ndd-zeroing or shred, which can take some time to run. Finally, the user is free to create new\ncleanup scripts of their own and have them specified in the configmap of the provisioner.\n\nThe configmap from earlier section will be enhanced as follows\n```\ndata:\n  storageClassMap: |\n    local-fast:\n      hostDir: \"/mnt/disks\"\n      mountDir: \"/local-ssds\"\n      blockCleanerCommand:\n         - \"/scripts/dd_zero.sh\"\n         - \"2\"\n ```\n\nThe block cleaner command will specify the script and any arguments that need to be passed to it.\nThe actual block device being cleaned will be supplied to the script as an environment variable\n(LOCAL_PV_BLKDEVICE) as opposed to command line, so that the script command line has complete\nfreedom on its structure. The provisioner will validate that the block device path is actually\nwithin the directory managed by the provisioner, to prevent destructive operations on arbitrary\npaths.\n\nThe provisioner logic currently does each volume’s cleanup as a synchronous serial activity.\nHowever, with cleanup now potentially being a multi hour activity, the processes will have to\nbe asynchronous and capable of being executed in parallel. The provisioner will ensure that all\ncurrent asynchronous cleanup processes are tracked. Special care needs to be taken to ensure that\nwhen a disk has only been partially cleaned. This scenario can happen if some impatient user\nmanually deletes a PV and the provisioner ends up re-creating pv ready for use (but only partially\ncleaned). This issue will be addressed in the re-design of the provisioner (details will be provided\nin the re-design section). The re-design will ensure that all disks being cleaned will be tracked\nthrough custom resources, so no disk being cleaned will be re-created as a PV.\n\nThe provisioner will also log events to let the user know that cleaning is in progress and it can\ntake some time to complete.\n\n### Risks and Mitigations\nThere are some major risks of using this feature:\n\n * A pod's availability becomes tied to the node's availability. If the node where the local volume is\nlocated at becomes unavailable, the pod cannot be rescheduled since it's tied to that node's data.\nUsers must be aware of this limitation and design their applications accordingly. Recovery from this\nkind of failure can be manual or automated with an operator tailored to the application's recovery process.\n* The underlying backing disk has its own varying durability guarantees that users must understand.\nFor example, in many cloud environments, local disks are ephemeral and all data can be lost at any time.\nJust because we call it \"PersistentVolume\" in Kubernetes doesn't mean the underlying backing store provides\nstrong data durability.\n\n## Test Plan\n\n### API unit tests\n\n* LocalVolumeSource cannot be specified without the feature gate\n* Non-empty PV node affinity is required for LocalVolumeSource\n* Preferred node affinity is not allowed\n* Path is required to be non-empty\n* Invalid json representation of type NodeAffinity returns error\n\n### PV node affinity unit tests\n\n* Nil or empty node affinity evaluates to true for any node\n* Node affinity specifying existing node labels evaluates to true\n* Node affinity specifying non-existing node label keys evaluates to false\n* Node affinity specifying non-existing node label values evaluates to false\n\n### Local volume plugin unit tests\n\n* Plugin can support PersistentVolumeSource\n* Plugin cannot support VolumeSource\n* Plugin supports ReadWriteOnce access mode\n* Plugin does not support remaining access modes\n* Plugin supports Mounter and Unmounter\n* Plugin does not support Provisioner, Recycler, Deleter\n* Plugin supports readonly\n* Plugin GetVolumeName() returns PV name\n* Plugin ConstructVolumeSpec() returns PV info\n* Plugin disallows backsteps in the Path\n\n### Local volume provisioner unit tests\n\n* Directory not in the cache and PV should be created\n* Directory is in the cache and PV should not be created\n* Directories created later are discovered and PV is created\n* Unconfigured directories are ignored\n* PVs are created with the configured StorageClass\n* PV name generation hashed correctly using node name, storageclass and filename\n* PV creation failure should not add directory to cache\n* Non-directory type should not create a PV\n* PV is released, PV should be deleted\n* PV should not be deleted for any other PV phase\n* PV deletion failure should not remove PV from cache\n* PV cleanup failure should not delete PV or remove from cache\n* Validating that a discovery directory containing both block and file system volumes are appropriately discovered and have PVs created.\n* Validate that both success and failure of asynchronous cleanup processes are properly tracked by the provisioner\n* Ensure a new PV is not created while cleaning of volume behind the PV is still in progress\n* Ensure two simultaneous cleaning operations on the same PV do not occur\n\n### E2E tests\n\n* Pod that is bound to a Local PV is scheduled to the correct node\nand can mount, read, and write\n* Two pods serially accessing the same Local PV can mount, read, and write\n* Two pods simultaneously accessing the same Local PV can mount, read, and write\n* Test both directory-based Local PV, and mount point-based Local PV\n* Launch local volume provisioner, create some directories under the discovery path,\nand verify that PVs are created and a Pod can mount, read, and write.\n* After destroying a PVC managed by the local volume provisioner, it should cleanup\nthe volume and recreate a new PV.\n* Pod using a Local PV with non-existent path fails to mount\n* Pod that sets nodeName to a different node than the PV node affinity cannot schedule.\n* Validate block PV are discovered and created\n* Validate cleaning of released block PV using each of the block cleaning scripts included.\n* Validate that file and block volumes in the same discovery path have correct PVs created, and that they are appropriately cleaned up.\n* Leverage block PV via PVC and validate that serially writes data in one pod, then reads and validates the data from a second pod.\n* Restart of the provisioner during cleaning operations, and validate that the PV is not recreated by the provisioner until cleaning has occurred.\n\n### Stress tests\n\n* Create a few hundred local PVs and even more Pods, where each pod specifies a varying number of PVCs.\nRandomly create and delete Pods and their PVCs at varying intervals. All Pods should be schedulable as\nPVs get recycled. Test with and without the static provisioner.\n\n## Graduation Criteria\n\n### Alpha -\u003e Beta\n* Basic unit and e2e tests as outlined in the test plan.\n* Metrics in k/k for volume mount/unmount, device mount/unmount operation\n  latency and error rates.\n* Metrics in local static provisioner for discovery and deletion operation\n  latency and error rates.\n\n### Beta -\u003e GA\n* Stress tests to iron out possible race conditions in the scheduler.\n* Users deployed in production and have gone through at least one K8s upgrade.\n\n## Implementation History\n\n### K8s 1.7: Alpha\n\n* Adds a `local` PersistentVolume source that allows specifying a directory\nor mount point with node affinity as an alpha annotation.\n* Limitations:\n    * Does not support specifying multiple local PVCs in a Pod.\n    * PVC binding does not consider pod scheduling requirements and may make suboptimal or incorrect decisions.\n\n### K8s 1.9: Alpha\n\nStill alpha, but with improved scheduler support\n\n* A new StorageClass `volumeBindingMode` parameter was added that\nenables delaying PVC binding until a pod is scheduled. This addresses the limitations from 1.7.\n\n### K8s 1.10: Beta\n\n* `NodeAffinity` beta field was added to PersistentVolume, and the alpha annotation was deprecated.\n    * A [one-time job](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner/blob/master/utils/update-pv-to-beta)\nwas added to help users migrate from the alpha annotation to the beta field.\n* Raw block alpha support was added specified by PV.volumeMode = `Block`.\n\n### K8s 1.12: Beta\n\n* If PV.volumeMode = `Filesystem` but the local volume path was a block device, then Kubernetes will automatically\nformat the device with the filesystem type specified in `FSType`.\n\n## Infrastructure Needed\n\n* A new repository at [kubernetes-sigs/sig-storage-local-static-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner)\nis needed to develop the static provisioner.\n* Build local static provisioner container images using prow.\n* Prow CI jobs for the local static provisioner.\n"
+  },
+  {
+    "id": "5fc7bff787524067263a7395d19f317c",
+    "title": "Online Growing Persistent Volume Size",
+    "authors": ["@mlmhl", "@wongma7"],
+    "owningSig": "sig-storage",
+    "participatingSigs": ["sig-storage"],
+    "reviewers": ["@gnufied", "@jsafrane"],
+    "approvers": ["@childsb"],
+    "editor": "TBD",
+    "creationDate": "2019-01-25",
+    "lastUpdated": "2019-02-01",
+    "status": "implementable",
+    "seeAlso": [
+      "https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/grow-volume-size.md",
+      "https://github.com/kubernetes/community/pull/1535"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Online Growing Persistent Volume Size\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Story 1](#story-1)\n    - [Story 2](#story-2)\n  - [Notes](#notes)\n  - [Implementation Details](#implementation-details)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Test Plan](#test-plan)\n- [Graduation Criteria](#graduation-criteria)\n  - [Alpha to Beta](#alpha-to-beta)\n  - [Beta to GA](#beta-to-ga)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis feature enables users to expand a volume's file system by editing a PVC without having to restart a pod using the PVC.\n\n## Motivation\n\nRelease 1.10 only supports offline file system resizing for PVCs, as this operation is only executed inside the `MountVolume` operation in kubelet. If a resizing request was submitted after the volume was mounted, it won't be performed. This proposal's intent is to support online file system resizing for PVCs in kubelet.\n\n### Goals\n\nEnable users to increase the size of a PVC which is already in use (mounted). The user will update PVC to request a new size. Underneath we expect that kubelet will resize the file system for the PVC accordingly.\n\n### Non-Goals\n\n* Offline file system resizing is not included. If we find a volume needs file system resizing but is not mounted to the node yet, we will do nothing. This situation will be dealt with by the existing [offline file system resizing handler](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/grow-volume-size.md).\n\n* Extending resize tools: we only support the most common file systems' offline resizing in current release, and we prefer to stay the same for online resizing: ext3, ext4, \u0026 xfs.\n\n## Proposal\n\n### User Stories\n\n#### Story 1\n\n* As a user I am running MySQL on a 100GB volume - but I am running out of space. I should be able to increase size of volume MySQL is using without losing all my data. (online and with data)\n\n#### Story 2\n\n* As a user I am running an application with a PVC. I should be able to resize the volume without losing data or mount point. (online and with data and without taking pod offline)\n\n### Notes\n\n- Currently we only support offline resizing for `xfs`, `ext3`, `ext4`. Online resizing of `ext3`,`ext4` was introduced in [Linux kernel-3.3](https://www.ibm.com/developerworks/library/l-33linuxkernel/), and `xfs` has always supported growing mounted partitions (in fact, currently there is no way to expand an unmounted `xfs` file system), so they are all safe for online resizing. If a user tries to expand a volume with other formats an error event will be reported for the pod using it.\n\n- This feature is protected by an alpha feature gate `ExpandOnlinePersistentVolumes` in v1.11. We separate this feature gate from the offline resizing gate `ExpandPersistentVolumes`, if a user wants to enable this feature, `ExpandPersistentVolumes` must be enabled first.\n\n### Implementation Details\n\nThe key point of online file system resizing is how kubelet discovers which PVCs need file system resizing. We achieve this goal by reusing the reprocess mechanism of `VolumeManager`'s `DesiredStateOfWorldPopulator`. kubelet synchronizes pods periodically, and during each loop, `DesiredStateOfWorldPopulator` is called to reprocess each pod's volumes, where it ensures they are in `DesiredStateOfWorld`.\n\nWhen adding a volume to `DesiredStateOfWorld`, the populator will include the PV \u0026 PVC fields needed to check whether the volume requires an online resizing operation: one is required if `PVC.Status.Capacity` is less than `PV.Spec.Capacity`.\n\nLater, `VolumeManager`'s `Reconciler` mounts the volume in `DesiredStateOfWorld` to a pod and adds it to `ActualStateOfWorld`. It will copy the `PVC.Status.Capacity` field from the `DesiredStateOfWorld` representation to the `ActualStateOfWorld` one.\n\nThe reconciler will periodically check whether any volume that's mounted in a pod (as reported by `ActualStateOfWorld`) requires an online resizing operation by reading its fields and triggering an online file system resize operation if: the volume's `PVC.Status.Capacity` in `ActualStateOfWorld` is less than its `PV.Spec.Capacity` in `DesiredStateOfWorld`, the volume has `volumeMode` `Filesystem`, and the volume is not `readOnly`.\n\nIf the file system resizing operation succeeds, `PVC.Status.Capacity` is changed to the desired volume size in `PV.Spec.Capacity`, and the `PersistentVolumeClaimFileSystemResizePending` condition removed from `PVC.Status.Conditions`. The `PVC.Status.Capacity` in `ActualStateOfWorld` is changed too. The `PVC.Status.Capacity` in `DesiredStateOfWorld` will be changed by the populator the next time it reprocesses the pod. (If the same volume is mounted again to another pod before that, a no-op resize may be triggered).\n\nIf it fails, the reconciler retries.\n\nIt is important to note that:\n\n- The reconciler triggers resize operations only for volumes that are mounted to pods by reading `ActualStateOfWorld`, which should only be read/written to by reconciler and the operations it starts. operationExecutor's goroutinemap prevents resize operations and other operations (e.g. unmount) from happening simultaneously. \n\n- File system resizing is a global operation for a volume, so if more than one pod mounts the same PVC, the reconciler should avoid triggering unnecessary no-op operations. (The alpha implementation used an in-memory map but it might also be possible to change `PVC.Status.Capacity` for every `mountedPod` of an `attachedVolume`.)\n\n- Since we support only `xfs` and `ext3/4`, we needn't worry about exotic file systems that can be attached/mounted to multiple nodes at the same time and require a resizing operation on a node, such as [gfs2](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/global_file_system_2/s1-manage-growfs).\n\n### Risks and Mitigations\n\n- All in-tree volume plugins that support offline file system expansion support online file system expansion:\n  - GCE PD\n  - Azure Disk\n  - AWS EBS\n  - RBD\n  - Cinder\n\nHowever, Azure Disk in particular does not support online \"device\" expansion, so in practice a pod needs to be restarted and the device reattached for resize to happen regardless of whether this feature is enabled. File system expansion follows \"device\"/\"controller\"/\"cloud provider\" expansion (i.e. file system expansion is the resize2fs call to enlarge a file system to fill a device, and device expansion is the API call to expand the device in the first place). This difference is surfaced to the user via the \"FileSystemResizePending\" and \"Resizing\" conditions, for when file system or cloud provider expansion are in progress respectively. No additional conditions or configuration options will be surfaced to account for this case of online device expansion being unsupported: it is left to the volume plugin implementor \u0026 cloud provider to document it and return meaningful errors, which are displayed as events on PVCs.\n\n- Offline resize was introduced first and requires a pod restart to take effect, so online resize becoming the default and occurring immediately upon a PVC edit without requiring a restart may come as a surprise to users. But expansion of mounted file systems is supported by the kernel, so all applications will tolerate it. Plus, if there are cases where users would prefer their volumes to not resize immediately, they can still force offline resize to happen by stopping their pod before editing the PVC, then restarting it. Online resizing will be made default.\n\n- Enabling support of volume expansion through stateful sets (https://github.com/kubernetes/enhancements/issues/661) is orthogonal to making online resizing default. Stateful apps/workloads may need to restart to detect file systems' size changes but should have no issue with online resize.\n\n## Test Plan\n\nThere will be e2e tests for online resizing. To isolate vs offline resize, the test will edit the PVC while the Pod is started instead of before.\n\n## Graduation Criteria\n\n### Alpha to Beta\n\n- e2e test where the PVC is edited while the Pod is started instead of before\n- update existing e2e tests so that they test offline resize correctly whether the online resize feature is enabled or not.\n- rename the operation metric to \"online_node_expansion\"\n\n### Beta to GA\n\n- Time for feedback (at least 1 release)\n- stress tests where multiple Pods are started/stopped \u0026 PVCs edited simultaneously to find race conditions in volume manager\n\n## Implementation History\n\n- 1.11: alpha\n- 1.15: beta\n"
+  },
+  {
+    "id": "4929c2ee8be54f72a2e0ca723edc3c62",
+    "title": "In-tree Storage Plugin to CSI Migration",
+    "authors": ["@davidz627", "@jsafrane"],
+    "owningSig": "sig-storage",
+    "participatingSigs": ["sig-architecture", "sig-cluster-lifecycle"],
+    "reviewers": ["@saadali", "@msau42"],
+    "approvers": ["@saadali"],
+    "editor": "@davidz627",
+    "creationDate": "2019-01-29",
+    "lastUpdated": "2019-01-29",
+    "status": "implementable",
+    "seeAlso": [
+      "https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/csi-migration.md"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# In-tree Storage Plugin to CSI Migration Design Doc\n\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n  - [Glossary](#glossary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis document presents a detailed design for migrating in-tree storage plugins\nto CSI. This will be an opt-in feature turned on at cluster creation time that\nwill redirect in-tree plugin operations to a corresponding CSI Driver.\n\n### Glossary\n\n* ADC (Attach Detach Controller): Controller binary that handles Attach and Detach portion of a volume lifecycle\n* Kubelet: Kubernetes component that runs on each node, it handles the Mounting and Unmounting portion of volume lifecycle\n* CSI (Container Storage Interface): An RPC interface that Kubernetes uses to interface with arbitrary 3rd party storage drivers\n* In-tree: Code that is compiled into native Kubernetes binaries\n* Out-of-tree: Code that is not compiled into Kubernetes binaries, but can be run as Deployments on Kubernetes\n\n\n## Motivation\n\nThe Kubernetes volume plugins are currently in-tree meaning all logic and\nhandling for each plugin lives in the Kubernetes codebase itself. With the\nContainer Storage Interface (CSI) the goal is to move those plugins out-of-tree.\nCSI defines a standard interface for communication between the Container\nOrchestrator (CO), Kubernetes in our case, and the storage plugins.\n\nAs the CSI Spec moves towards GA and more storage plugins are being created and\nbecoming production ready, we will want to migrate our in-tree plugin logic to\nuse CSI plugins instead. This is motivated by the fact that we are currently\nsupporting two versions of each plugin (one in-tree and one CSI), and that we\nwant to eventually transition all storage users to CSI.\n\nIn order to do this we need to migrate the internals of the in-tree plugins to\ncall out to CSI Plugins because we will be unable to deprecate the current\ninternal plugin API’s due to Kubernetes API deprecation policies. This will\nlower cost of development as we only have to maintain one version of each\nplugin, as well as ease the transition to CSI when we are able to deprecate the\ninternal APIs.\n\n### Goals\n\n* Compile all requirements for a successful transition of the in-tree plugins to\n  CSI\n    * As little code as possible remains in the Kubernetes Repo\n    * In-tree plugin API is untouched, user Pods and PVs continue working after\n      upgrades\n    * Minimize user visible changes\n* Design a robust mechanism for redirecting in-tree plugin usage to appropriate\n  CSI drivers, while supporting seamless upgrade and downgrade between new\n  Kubernetes version that uses CSI drivers for in-tree volume plugins to an old\n  Kubernetes version that uses old-fashioned volume plugins without CSI.\n* Design framework for migration that allows for easy interface extension by\n  in-tree plugin authors to “migrate” their plugins.\n    * Migration must be modular so that each plugin can have migration turned on\n      and off separately\n\n### Non-Goals\n\n* Design a mechanism for deploying  CSI drivers on all systems so that users can\n  use the current storage system the same way they do today without having to do\n  extra set up.\n* Implementing CSI Drivers for existing plugins\n* Define set of volume plugins that should be migrated to CSI\n\n## Proposal\n\n### Implementation Details/Notes/Constraints\nThe detailed design was originally implemented as a [design proposal](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/csi-migration.md)\n\n### Risks and Mitigations\n\n* Performance risks as outlined in design proposal\n* ADC and Kubelet synchronization fairly complicated, upgrade path non-trivial - mitigation discussed in design proposal\n\n## Graduation Criteria\n\n### Alpha -\u003e Beta\n\n* All volume operation paths covered by Migration Shim in Alpha for \u003e= 1 quarter\n* Tests outlined in design proposal implemented\n* Required CRD and driver installation solved generally\n\n### Beta -\u003e GA\n\n* All volume operation paths covered by Migration Shim in Beta for \u003e= 1 quarter without significant issues\n\n## Implementation History\n\nMajor milestones in the life cycle of a KEP should be tracked in `Implementation History`.\nMajor milestones might include\n\n- 2019-01-29 KEP Created\n- 2019-01-05 Implementation started\n"
+  },
+  {
+    "id": "48825c96b8dd5fe199bfd9001a38db79",
+    "title": "CSI Pod Info on Mount",
+    "authors": ["@jsafrane"],
+    "owningSig": "sig-storage",
+    "participatingSigs": ["sig-storage"],
+    "reviewers": ["@msau42", "@saad-ali"],
+    "approvers": ["@saad-ali"],
+    "editor": "TBD",
+    "creationDate": "2019-01-29",
+    "lastUpdated": "2019-01-29",
+    "status": "implementable",
+    "seeAlso": [
+      "https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/container-storage-interface-pod-information.md"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# CSI Pod Info on Mount\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Test Plan](#test-plan)\n- [Graduation Criteria](#graduation-criteria)\n  - [Alpha to Beta](#alpha-to-beta)\n  - [Beta to GA](#beta-to-ga)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis document presents a design to allow Kubernetes to pass metadata such as Pod name and namespace to the CSI `NodePublishVolume` call if a CSI driver requires it.\n\nThe detailed design was originally implemented as a [design proposal](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/container-storage-interface-pod-information.md).\n\nThis KEP contains details that are missing from the design proposal.\n\n## Test Plan\n\n* Unit tests in kubelet volume manager.\n* E2E tests:\n    * `CSI workload information [Feature:CSIDriverRegistry]` via CSI mock driver \n\n## Graduation Criteria\n\n### Alpha to Beta\n\n* Basic unit and e2e tests as outlined in the test plan.\n\n### Beta to GA\n\n* At least one CSI driver implemented using this feature in production.\n\n## Implementation History\n\n* K8s 1.12: Alpha implementation\n* K8s 1.14: Beta implementation\n* K8s 1.18: GA implementation\n"
+  },
+  {
+    "id": "ed69f84b853ad416112ea8c5e79e54be",
+    "title": "Skip attach for non-attachable CSI volumes",
+    "authors": ["@jsafrane"],
+    "owningSig": "sig-storage",
+    "participatingSigs": ["sig-storage"],
+    "reviewers": ["@msau42", "@saad-ali"],
+    "approvers": ["@saad-ali"],
+    "editor": "TBD",
+    "creationDate": "2019-01-29",
+    "lastUpdated": "2019-01-29",
+    "status": "implementable",
+    "seeAlso": [
+      "https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/container-storage-interface-skip-attach.md"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Skip attach for non-attachable CSI volumes\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Test Plan](#test-plan)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis document presents a design to be able to skip the attach/detach flow in\nKubernetes for CSI plugins that don't support attaching.\n\nThe detailed design was originally implemented as a [design\nproposal](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/container-storage-interface-skip-attach.md).\n\nThis KEP contains details that are missing from the design proposal.\n\n## Test Plan\n* Unit tests in attach detach controller\n* Integration tests:\n   * A VolumeAttachment object is not created for CSI drivers that don't\n     support attach\n   * A VolumeAttachment object is created for CSI drivers that do\n     support attach\n   * A VolumeAttachment object is created for CSI drivers that did not\n     specify attach support\n* E2E tests:\n    * Drivers that don't support attach don't need the external-attacher and can\n      mount volumes successfully\n    * Drivers that don't require attach but has missing CSIDriver objects should successfully\n      mount volumes without attachment(and no external-attacher running) if CSIDriver\n      object is created later on.\n\n## Graduation Criteria\n\n### Alpha -\u003e Beta\n* Basic unit and e2e tests as outlined in the test plan.\n\n### Beta -\u003e GA\n* Users deployed in production and have gone through at least one K8s upgrade.\n\n## Implementation History\n* K8s 1.12: Alpha implementation\n* K8s 1.14: Beta implementation\n* K8s 1.18: GA implementation\n"
+  },
+  {
+    "id": "63c982578546ff830332ddd82f1f8364",
+    "title": "Support for CSI volume resizing",
+    "authors": ["@gnufied"],
+    "owningSig": "sig-storage",
+    "participatingSigs": ["sig-storage"],
+    "reviewers": ["@saad-ali", "@jsafrane"],
+    "approvers": ["@saad-ali", "@childsb"],
+    "editor": "",
+    "creationDate": "2019-01-29",
+    "lastUpdated": "2019-01-29",
+    "status": "implementable",
+    "seeAlso": [
+      "[Kubernetes Volume expansion](https://github.com/kubernetes/enhancements/issues/284)",
+      "[Online resizing design](https://github.com/kubernetes/enhancements/pull/737)"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Support for CSI volume resizing\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [External resize controller](#external-resize-controller)\n  - [Expansion on Kubelet](#expansion-on-kubelet)\n    - [Offline volume resizing on kubelet:](#offline-volume-resizing-on-kubelet)\n    - [Online volume resizing on kubelet:](#online-volume-resizing-on-kubelet)\n    - [Supporting per-PVC secret refs](#supporting-per-pvc-secret-refs)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Test Plan](#test-plan)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nTo bring CSI volumes in feature parity with in-tree volumes we need to implement support for resizing of CSI volumes.\n\n## Motivation\n\nWe recently implemented volume resizing support in CSI specs. This proposal implements this feature for Kubernetes.\nAny CSI volume plugin that implements necessary part of CSI specs will become resizable.\n\n### Goals\n\nTo enable expansion of CSI volumes used by `PersistentVolumeClaim`s that support volume expansion as a plugin capability.\n\n### Non-Goals\n\nThe expansion capability of a CSI plugin will not be validated by using CSI RPC call when user edits the PVC(i.e existing resize admission controller will not make CSI RPC call).\nThe responsibility of\nactually enabling expansion for certains storageclasses still falls on Kubernetes admin.\n\n## Proposal\n\nThe design of CSI volume resizing is made of two parts.\n\n\n### External resize controller\n\nTo support resizing of CSI volumes an external resize controller will monitor all changes to PVCs.  If a PVC meets following criteria for resizing, it will be added to\ncontroller's workqueue:\n\n- The driver name disovered from PVC(via corresponding PV) should match name of driver currently known(by querying driver info via CSI RPC call) to external resize controller.\n- Once it notices a PVC has been updated and by comparing old and new PVC object, it determines more space has been requested by the user.\n\nOnce PVC gets picked from workqueue, the controller will also compare requested PVC size with actual size of volume in `PersistentVolume`\nobject. Once PVC passes all these checks, a CSI `ControllerExpandVolume` call will be made by the controller if CSI plugin implements `ControllerExpandVolume`\nRPC call.\n\nIf `ControllerExpandVolume` call is successful and plugin implements `NodeExpandVolume`:\n- if `ControllerExpandVolumeResponse` returns `true` in `node_expansion_required` then `FileSystemResizePending` condition will be added to PVC and `NodeExpandVolume` operation will be queued on kubelet. Also volume size reported by PV will be updated to new value.\n- if `ControllerExpandVolumeResponse` returns `false` in `node_expansion_required` then volume resize operation will be marked finished and both `pvc.Status.Capacity` and `pv.Spec.Capacity` will report updated value.\n\nIf `ControllerExpandVolume` call is successful and plugin does not implement `NodeExpandVolume` call then volume resize operation will be marked as finished and both `pvc.Status.Capacity` and `pv.Spec.Capacity` will report updated value.\n\nIf `ControllerExpandVolume`  call fails:\n- Then PVC will retain `Resizing` condition and will have appropriate events added to the PVC.\n- Controller will retry resizing operation with exponential backoff, assuming it corrects itself.\n\nA general mechanism for recovering from resize failure will be implemented via: https://github.com/kubernetes/kubernetes/issues/73036\n\n### Expansion on Kubelet\n\nA CSI volume may require expansion on the node to finish volume resizing. In some cases - the entire resizing operation can happen on the node and\nplugin may choose to not implement `ControllerExpandVolume` CSI RPC call at all.\n\nCurrently Kubernetes supports two modes of performing volume resize on kubelet. We will describe each mode here. For more information , please refer to original volume resize proposal - https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/grow-volume-size.md.\n\n\n#### Offline volume resizing on kubelet:\n\nThis is the default mode and in this mode `NodeExpandVolume` will only be called when volume is being mounted on the node(or `MountVolume` operation in `operation_executor.go`). In other words, pod that was using the volume must be re-created for expansion on node to happen.\n\nWhen a pod that is using the PVC is started, kubelet will compare `pvc.spec.resources.requests.storage` and `pvc.Status.Capacity`. It also compares PVC's size with `pv.Spec.Capacity` and if it detects PV is reporting same size as pvc's spec but PVC's status is still reporting smaller value then it determines -\na volume expansion is pending on the node.  At this point if plugin implements `NodeExpandVolume` RPC call then, kubelet will call it and:\n\nIf `NodeExpandVolume` is successful:\n- It will update `pvc.Status.Capacity` with latest value and remove all resizing related conditions from PVC.\n\nIf `NodeExpandVolume` failed:\n- It will add a event to both PVC and Pod about failed resizing and resize operation will be retried. This will prevent pod from starting up.\n\n\n#### Online volume resizing on kubelet:\n\nMore details about online resizing can be found in [Online resizing design](https://github.com/kubernetes/enhancements/pull/737) but essentially if\n`ExpandInUsePersistentVolumes` feature is enabled then kubelet will periodically poll all PVCs that are being used on the node and compare `pvc.spec.resources.requests.storage` and `pvc.Status.Capacity`(also `pv.Spec.Capacity`) and make similar determination about whether node expansion is required for the volume.\n\nIn this mode `NodeExpandVolume` can be called while pod is running and volume is in-use. Using aformentioned check if kubelet determines that\nvolume expansion is needed on the node and plugin implements `NodeExpandVolume` RPC call then, kubelet will call it(provided volume has already been node staged and published on the node) and:\n\nIf `NodeExpandVolume` is successful:\n- It will update `pvc.Status.Capacity` with latest value and remove all resizing related conditions from PVC.\n\nIf `NodeExpandVolume` failed:\n- It will add a event to both PVC and Pod about failed resizing and resize operation will be retried.\n\n#### Supporting per-PVC secret refs\n\nTo support per-PVC secrets for volume resizing, similar to CSI attach and detach - this proposal expands `CSIPersistentVolumeSource` object to contain `ControllerExpandSecretRef`. This API change will be gated by `ExpandCSIVolumes` feature gate currently in Alpha:\n\n```\ntype CSIPersistentVolumeSource struct {\n    ....\n    // ControllerPublishSecretRef is a reference to the secret object containing\n    // sensitive information to pass to the CSI driver to complete the CSI controller publish\n    ControllerPublishSecretRef *SecretReference\n\n    // ControllerExpandSecretRef is a reference to secret object containing sensitive\n    // information to pass to the CSI driver to complete CSI controller expansion\n    ControllerExpandSecretRef *SecretReference\n}\n```\n\nSecrets will be fetched from StorageClass with parameters `csi.storage.k8s.io/controller-expand-secret-name` and `csi.storage.k8s.io/controller-expand-secret-namespace`. Resizing secrets will support same templating rules as attach and detach as documented - https://kubernetes-csi.github.io/docs/secrets-and-credentials.html#controller-publishunpublish-secret .\n\nStarting from 1.15 it is expected that all CSI volumes that require secrets for expansion will have `ControllerExpandSecretRef` field set. If not set\n`ControllerExpandVolume` CSI RPC call will be made without secret. Existing validation of `PersistentVolume` object will be relaxed to allow\nsetting of `ControllerExpandSecretRef` for the first time so as CSI volume expansion can be supported for existing PVs.\n\nA similar field for `NodeExpandVolume` RPC call is not required because CSI `NodeExpandVolume` does not accepts secrets. It is also expected that\nKubelet will not require access to `ControllerExpandSecretRef` field.\n\n### Risks and Mitigations\n\nBefore this feature goes GA - we need to handle recovering https://github.com/kubernetes/kubernetes/issues/73036.\n\n## Test Plan\n\n* Unit tests for external resize controller.\n* Add e2e tests in Kubernetes that use csi-mock driver for volume resizing.\n  - (positive) Give a plugin that supports both control plane and node size resize, CSI volume should be resizable and able to complete successfully.\n  - (positive) Given a plugin that only requires control plane resize, CSI volume should be resizable and able to complete successfully.\n  - (positive) Given a plugin that only requires node side resize, CSI volume should be resizable and able to complete successfully.\n  - (positive) Given a plugin that support online resizing, CSI volume should be resizable and online resize operation be able to complete successfully.\n  - (negative) If control resize fails, PVC should have appropriate events.\n  - (neative) if node side resize fails, both pod and PVC should have appropriate events.\n\n## Graduation Criteria\n\nOnce implemented CSI volumes should be resizable and in-line with current in-tree implementation of volume resizing.\n\n* *Alpha* :\n  - Kubernetes - 1.14:  Initial support for CSI volume resizing. Released code will include an external CSI volume resize controller and changes to Kubelet. Implementation will have unit tests and csi-mock driver e2e tests.\n  - Kubernetes - 1.15:  Add e2e tests that use real drivers(`gce-pd`, `ebs` at minimum). Add metrics for volume resize operations. Support per-PVC secret refs.\n* *Beta*  : More robust support for CSI volume resizing, handle recovering from resize failures.\n* *GA* : CSI resizing in general will only leave GA after existing [Volume expansion](https://github.com/kubernetes/enhancements/issues/284) feature leaves GA. Online resizing of CSI volumes depends on [Online resizing](https://github.com/kubernetes/enhancements/pull/737) feature and online resizing of CSI volumes will be available as a GA feature only when [Online resizing feature](https://github.com/kubernetes/enhancements/pull/737) goes GA.\n\nHopefully the content previously contained in [umbrella issues][] will be tracked in the `Graduation Criteria` section.\n\n[umbrella issues]: https://github.com/kubernetes/kubernetes/issues/62096\n\n## Implementation History\n\n- 1.14 Implement CSI volume resizing as an alpha feature.\n- 1.11 Move in-tree volume expansion to beta.\n- 1.11 Implement online resizing feature for in-tree volume plugins as an alpha feature.\n- 1.8 Implement in-tree volume expansion an an alpha feature.\n"
+  },
+  {
+    "id": "9511b0a643c1ccea5114889c49d556fe",
+    "title": "CSI Raw Block Volumes",
+    "authors": ["@bswartz"],
+    "owningSig": "sig-storage",
+    "participatingSigs": ["sig-storage"],
+    "reviewers": ["@msau42", "@saad-ali"],
+    "approvers": ["@saad-ali"],
+    "editor": "TBD",
+    "creationDate": "2019-01-30",
+    "lastUpdated": "2019-10-15",
+    "status": "implementable",
+    "seeAlso": [
+      "https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/raw-block-pv.md",
+      "https://github.com/kubernetes/enhancements/pull/1288"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# CSI Raw Block Volumes\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n- [Upgrade/Downgrade Strategy](#upgradedowngrade-strategy)\n- [Test Plan](#test-plan)\n  - [Unit tests](#unit-tests)\n  - [E2E tests](#e2e-tests)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n  - [K8s 1.12: Alpha](#k8s-112-alpha)\n  - [K8s 1.13: Alpha](#k8s-113-alpha)\n  - [K8s 1.14: Beta](#k8s-114-beta)\n  - [K8s 1.17: Beta](#k8s-117-beta)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nRaw block support has been added to both the core of Kubernetes (including\nAPI support and support in kubelet for a subset of volume types) as well as\nthe CSI specification. This feature is specifically to add raw block support\nto kubelet for volumes of the \"csi\" type by taking advantage of the\nnow-standardized CSI RPCs.\n\nhttps://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/raw-block-pv.md\n\n## Motivation\n\nThe future of persistent volume support should be mostly CSI-type volumes,\nwith a few notable exceptions. It's therefore essential that CSI volumes\ncan implement all of the features that Kubernetes has, including support\nfor raw block volumes.\n\n### Goals\n\nCSI plugins should be able to create, publish, and attach raw block volumes\nin accordance with the raw block support APIs.\n\n### Non-Goals\n\nRaw block support added in any particular CSI driver.\n\n## Proposal\n\nModify the CSI volume package of kubelet to declare support for raw block\nvolumes, and make the following changes to the csi mounter plugin:\n\nFor raw block volumes:\n* Inside MountDevice, invoke NodeStageVolume() without a fsType, but with an\nempty directory (just like file system volumes).\n* Inside SetUp, invoke NodePublishVolume() with a device file name rather than\na directory name, and do not pass mount options, and expect the device to be\ncreated by the node plugin. Kubelet ensures the parent directory of the file\nname exists.\n* The volume is passed to the container layer as an actual device rather than\na mount.\n* On cleanup kubelet removes the directories it created, and assumes the node\nplugin removed any device files it created.\n\nThis support is controlled by a feature gate called \"CSIBlockVolume\".\n\n## Upgrade/Downgrade Strategy\n\nCSIBlockVolume feature gate is used only in CSI volume plugin and only in\nthe parts that are used by kubelet (`BlockVolumeMapper` interface). A node\nmust be therefore drained before switching the feature off to remove all\nraw devices provided to pods, because newly started kubelet (with the feature\noff) won't be able to remove these devices.\n\nWhen a node is not drained and CSIBlockVolume is disabled while running\na pod, we make sure that the pod is either killed or can continue\nusing the volume as before. In both cases, kubelet won't touch data\non the volume and can't corrupt it. The volume may not be cleaned after\nthe pod is deleted, i.e. some leftover symlinks / bind mounts may be\npresent. It's up to the cluster admin to clean these orphans\n(or drain nodes properly before disabling the feature).\n\n## Test Plan\n\n### Unit tests\n\n* Kubelet can generate staging/publish paths to send to the CSI plugins\n* Test that fake CSI plugin stages and publishes to the correct paths\n* Ensure that paths with and without dots work\n* Test that teardown cleans up everything\n\n### E2E tests\n\nThere are existing e2e tests for the raw block volume feature. To test CSI raw\nblock volumes we just need to configure a CSI plugin that has raw block support,\nand run the existing raw block tests on that CSI plugin.\n\n## Graduation Criteria\n\n### Alpha -\u003e Beta\n* At least 1 CSI plugin that can run in the gate (hostpath or GCE-PD) supporting\nraw block volumes so the feature can be regression tested.\n* Enable e2e tests for raw block with CSI\n* At least 2 third party CSI plugins support raw block with CSI and pass e2e\ntests to confirm both that the interface is flexible enough to accommodate\nmultiple implementations and that it's specific enough to not leave any\nambiguity about how implementations should work.\n\n### Beta -\u003e GA\n* Two or more CSI plugins that and run in the gate and test raw block\n* At least 5 third party CSI plugins support raw block with CSI and pass e2e\ntests\n* Manual stress test with at least one real CSI driver is performed with a node\n  running non-trivial amount of pods that use a block device (simple `dd`\n  should do).\n  * Test that the pods are moved to another nodes and data is retained when\n    the node is drained.\n  * Test that the data is retained when kubelet restarts while the pods are\n    running.\n  * Test that the data is retained when the node reboots. For pods that were\n    moved to another nodes during the outage, test that the newly started node\n    cleaned up their devices.\n\n## Implementation History\n\n### K8s 1.12: Alpha\n* Initial implementation\n\n### K8s 1.13: Alpha\n* Multiple third party plugins start to implement the raw block CSI interface\nand we can test with them\n* Bugs fixed in CSI sidecars \n\n### K8s 1.14: Beta\n* Bugs fixed\n\n### K8s 1.17: Beta\n* Separated NodeStage / NodePublish calls.\n* Fixed volume reconstruction after kubelet restart.\n* Stress tests as noted above.\n"
+  },
+  {
+    "id": "0cf2cfa91c9f7edd77c99711fd9a00a3",
+    "title": "kubernetes-csi release process",
+    "authors": ["@pohly"],
+    "owningSig": "sig-storage",
+    "participatingSigs": ["sig-testing", "sig-release"],
+    "reviewers": ["@msau42"],
+    "approvers": ["@msau42", "sig-testing: TBD", "sig-release: TBD"],
+    "editor": "@pohly",
+    "creationDate": "2019-02-04",
+    "lastUpdated": "2019-02-04",
+    "status": "provisional",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# kubernetes-csi-release-process\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Versionioning](#versionioning)\n    - [Release artifacts](#release-artifacts)\n    - [Release process](#release-process)\n  - [Implementation Details](#implementation-details)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Drawbacks](#drawbacks)\n- [Alternatives](#alternatives)\n- [Infrastructure Needed](#infrastructure-needed)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThe Kubernetes [Storage\nSIG](https://github.com/kubernetes/community/tree/master/sig-storage)\nmaintains a set of components under the\n[kubernetes-csi](https://github.com/kubernetes-csi) GitHub\norganization. Those components are intentionally not part of core\nKubernetes even though they are maintained by the Kubernetes project.\n\nThis document explains how these components are released.\n\n## Motivation\n\nSo far, the process for tagging and building components has been\nfairly manual, with some assistance by Travis CI. There has been no\nautomatic testing of the release candidates on a Kubernetes cluster,\nneither as stand-alone deployment of individual components nor as full\nend-to-end (E2E) testing of several Kubernetes-CSI components. Testing\nthe stable releases of the components is included in the automatic\nKubernetes testing, but running those tests against pre-release\ncomponents had to be done manually.\n\nWith CSI support reaching GA in Kubernetes 1.13, it is time to define\nand follow a less work-intensive and more predictable approach.\n\n### Goals\n\n- define the release process for each component\n- unit testing as mandatory pre-submit check for all components\n- testing on Kubernetes as mandatory pre-submit check for components that get\n  deployed in a cluster\n\n### Non-Goals\n\n- a combined \"Kubernetes-CSI release\": each component gets released\n  separately.  It is the responsibility of a CSI driver maintainer\n  to pick and test sidecar releases for a combined deployment of that\n  driver. The [hostpath deployment\n  example](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy)\n  can be used as a starting point, but it's not going to be able to\n  test all the possible combinations of features that a specific CSI\n  driver may need.\n- change responsibilities among the kubernetes-csi maintainers: as before, each\n  component will have some main maintainer who is responsible for releasing updates\n  of that component\n- automatically generate release notes: this will be covered by a separate, future KEP\n\n\n## Proposal\n\n### Versionioning\n\nEach of the components has its own documentation, versioning and\nrelease notes. [Semantic versioning](https://semver.org/) with `v`\nprefix (i.e. `v1.2.3`) is used for components that provide a stable\nAPI, like for example:\n- external-attacher\n- external-provisioner\n- external-snapshotter\n- cluster-driver-registrar\n- node-driver-registrar\n- csi-driver-host-path\n- csi-lib-utils\n\nOther components are internal utility packages that are\nnot getting tagged:\n- csi-release-tools\n\nAn additional suffix like `-rc1` can be used to denote a pre-release.\n\nThe [hostpath example\ndeployment](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy)\ndefines the exact release of each sidecar that is part of that\ndeployment. When updating to newer sidecar releases, a new `csi-driver-host-path`\nrelease is tagged with version numbers bumped according to the\nsame rules as the individual components (major version bump when any\nof the sidecars had a major change, etc.). This will then also\nrebuild the hostpath driver binary itself with that version number\nembedded, regardless of whether its own source code has changed.\n\n#### Release artifacts\n\nTagging a component with a `v*` tag triggers a\nbuild for that component with that version. The output is primarily the container image\nfor components that need to be deployed. Those images get published as\n`gcr.io/kubernetes-csi/\u003cimage\u003e:\u003ctag\u003e`, which depends on [issue\n158](https://github.com/kubernetes/k8s.io/issues/158) getting resolved.\n\nOnly binaries provided as part of a release with a semantic version tag without additional suffix\nshould be considered\nproduction ready. Binaries are never going to be rebuilt, therefore an\nimage like `csi-node-driver-registrar:v1.0.2` will always pull the\nsame content and `imagePullPolicy: Always` can be omitted.\n\nEventually auxiliary files like the RBAC rules that are included in\neach component might also get published in a single location, like\n`https://dl.k8s.io/kubernetes-csi/`, but this currently outside of\nthis KEP.\n\n#### Release process\n\n1. A change is submitted against the master branch.\n\n1. A [Prow](https://github.com/kubernetes/test-infra/blob/master/prow/README.md)\n   job checks out the source of the modified component, rebuilds it and then\n   runs unit tests and E2E tests with it as defined below.\n\n1. Maintainers accept changes into the master branch.\n\n1. The same Prow job runs for master and repeats the checks. If it succeeds,\n   a new \"canary\" image is published for the component.\n\n1. In preparation for the release of a major new update, a feature freeze is\n   declared for the \"master\" branch and only changes relevant for that next\n   release are accepted.\n\n1. When all changes targeted for the release are in master, automatic\n   test results are okay and and potentially some more manual tests,\n   maintainers tag a new release directly on the master branch. This\n   can be a `-rc` test release or a normal release.\n\n1. Maintenance releases are prepared by creating a \"release-X.Y\" branch based on\n   release \"vX.Y\" and backporting relevant fixes from master. The same\n   prow job as for master also handles the maintenance branches.\n\n\n### Implementation Details\n\nFor each component under kubernetes-csi (`external-attacher`,\n`csi-driver-host-path`, `csi-lib-utils`, etc.), these Prow jobs need\nto be defined:\n- `kubernetes-csi-\u003ccomponent\u003e-pr`: presubmit job\n- `kubernetes-csi-\u003ccomponent\u003e-build`: a postsubmits job that matches against\n  `v*` branches *and* tags (see https://github.com/kubernetes/test-infra/pull/10802#discussion_r248900281)\n\nIn addition, for the `csi-driver-host-path` repo some more periodic\njobs need to be defined:\n- `kubernetes-csi-stable`: deploys and tests the current hostpath\n  example (`csi-driver-host-path/deploy/stable`) from the master\n  branch on the latest Kubernetes development version\n- `kubernetes-csi-canary-\u003ck8s-release\u003e`: deploys and tests the canary\n  hostpath example (`csi-driver-host-path/deploy/canary`) from the\n  master branch on a certain Kubernetes release (for example,\n  `\u003ck8s-release\u003e` = `1.13`), using the same image revisions for the\n  entire test run; initially we'll start with 1.13 and later will\n  add more stable releases and remove unsupported ones\n- `kubernetes-csi-canary-dev`: deploys and tests the canary hostpath\n  example from the master branch on the latest Kubernetes development\n  version\n\nA periodic job that does regular maintenance tasks (like checking for\nupdated dependencies) might be added in the future.\n\nThese `kubernetes-csi` Prow job all provide the same environment where\nthe component is already checked out in the `GOPATH` at the desired\nrevision (PR merged tentatively into a branch or a regular\nbranch). This is provided by the\n[podutils](https://github.com/kubernetes/test-infra/blob/master/prow/pod-utilities.md)\ndecorators. The base image is the latest\n[kubekins](https://github.com/kubernetes/test-infra/tree/master/images/kubekins-e2e)\nimage.\n\nThe Prow job transfers control to a `.prow.sh` shell script which must\nbe present in a component that is configured to trigger the Prow job.\n\nEach component has its own release configuration (what to build and\npublish) and rules (scripts, makefile). The advantage is that those\ncan be branched and tagged together with the component. The version of\nGo to use for building is also part of that configuration. The\nrequested version of Go will be installed from https://golang.org/dl/\nif different from what is installed already.\n\nTo simplify maintenance and ensure consistency, the common parts can\nbe shared via\n[csi-release-tools](https://github.com/kubernetes-csi/csi-release-tools/).\n\nUnit testing is provided by `make test`. Images are pushed with `make\npush`, which already determines image tags automatically. For Prow,\nthe image destination still needs to be determined\n(https://github.com/kubernetes/k8s.io/issues/158).\n\nFor testing on Kubernetes, a real cluster can be brought up with\n[kubetest](https://github.com/kubernetes/test-infra/tree/master/kubetest). This\nmight work with [kind](https://github.com/kubernetes-sigs/kind) and a\nlocally built image can be pushed directly into that cluster with\n`docker save image | docker exec -it kind-control-plane docker load\n-` (to be tested).\n\nA shared E2E test could work like this:\n- build one component from source\n- deploy the hostpath example with that locally build component and\n  everything else as defined in the current repository (i.e.\n  each repository must vendor, copy or check out the example from\n  `csi-drivers-host-path`)\n- check out a certain revision of the kubernetes repo and\n  run `go test ./test/e2e` with the parameter from https://github.com/kubernetes/kubernetes/pull/72836\n  to test the deployed example; alternatively, the test suite\n  can also be vendored, which would make the build self-contained\n  and allow extending the test suite\n\n\n### Risks and Mitigations\n\nPushing a new release image is triggered by setting a tag. Unless\nthere is a build failure, the result of the automatic build becomes\nthe latest release, without any further manual checking. There are\nmultiple risks:\n- automatic testing of a single component cannot catch all bugs, so\n  some bugs might make it into a tagged release\n- the wrong revision gets tagged by the maintainer\n- the build process itself is buggy and pushes a corrupt image\n\nTo mitigate this, maintainers can do a trial release first by tagging\na `-rc` version and doing additional manual tests with the result.\n\nBut ultimately the safeguard against such failures is that new CSI\nsidecar containers only get used in a production cluster after\npackagers of a CSI driver update the deployment files for their\ndriver.\n\n## Graduation Criteria\n\n- Test results are visible in GitHub PRs and test failures block merging.\n- Test results are visible in the [SIG-Storage\n  testgrid](https://k8s-testgrid.appspot.com/sig-storage-kubernetes) or\n  a sub-dashboard.\n- The Prow test output and/or metadata clearly shows what revisions of the\n  different components were tested.\n- All components have been converted to publishing images on `gcr.io` in addition\n  to the traditional `quay.io`.\n\n## Implementation History\n\n- 2019-02-04: initial draft\n\n## Drawbacks\n\nAllowing individual maintainers to create releases without going\nthrough a centralized release process implies that maintainers must be\nmore careful.\n\nBecause updated components get tested against the current set of other\ncomponents, breaking changes also break testing. If that ever becomes\nnecessary, manual intervention will be needed to release multiple\ndifferent component updates in sync.\n\n## Alternatives\n\nAutomatically updating deployments by setting additional image tags\nwas originally suggested in the [\"tag images using semantic\nversioning\" GitHub\nissue](https://github.com/kubernetes-csi/driver-registrar/issues/77),\nbut further discussion in [\"tag release images also with base\nversions\"](https://github.com/kubernetes-csi/csi-release-tools/issues/6)\nrejected that idea because of the risks associated with automatically\nchanging versions in a production cluster.\n\nA new hostpath driver binary gets created also when only the\ndeployment changes, for example because a sidecar gets updated. This\nis a result of treating the driver binary and its deployment as a unit\nwith a single version number. The alternative would have been to\nmaintain the deployment in a separate repository with its own\nindependent versioning, but that has been rejected because it is more\nwork and because driver and deployment always need to be updated\ntogether.\n\n## Infrastructure Needed\n\n* a Prow job that can start up test cluster(s), deploy images created as part of that job, and publish images\n"
+  },
+  {
+    "id": "61bd096680dc19469480ff73a0c19f6f",
+    "title": "Volume Scale and Performance Testing Plan",
+    "authors": ["@msau42"],
+    "owningSig": "sig-storage",
+    "participatingSigs": ["sig-scalability"],
+    "reviewers": ["@pohly", "@saad-ali", "@wojtek-t"],
+    "approvers": ["@saad-ali", "@wojtek-t"],
+    "editor": "TBD",
+    "creationDate": "2019-02-04",
+    "lastUpdated": "2019-03-01",
+    "status": "implementable",
+    "seeAlso": [
+      "https://github.com/kubernetes/community/blob/master/sig-scalability/slos/pod_startup_latency.md",
+      "https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/docs/design.md"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Volume Scale and Performance Testing Plan\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Test Framework](#test-framework)\n  - [Test Rollout](#test-rollout)\n  - [Test Portability](#test-portability)\n  - [Test Cases](#test-cases)\n    - [Pod Startup](#pod-startup)\n  - [WIP Future Test Cases](#wip-future-test-cases)\n    - [Pod Teardown](#pod-teardown)\n    - [PV Binding Tests](#pv-binding-tests)\n    - [PV Provisioning Tests](#pv-provisioning-tests)\n    - [PV Deletion Tests](#pv-deletion-tests)\n- [Graduation Criteria](#graduation-criteria)\n  - [Phase 1](#phase-1)\n  - [Phase 2](#phase-2)\n  - [Phase 3](#phase-3)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis KEP outlines a plan for testing scalability and performance of K8s storage components.\n\n## Motivation\n\nAdding storage scale and performance tests will help:\n* Understand the current scale limits of the Kuberentes storage system.\n* Set expectations (SLOs) for consumers of the Volume API.\n* Determine bottlenecks and influence which need addressing.\n\n### Goals\n\n* Measure the overhead of K8s components owned by sig-storage:\n  * K8s volume controllers\n  * Kubelet volume manager\n  * CSI sidecars\n* Stress various dimensions of volume operations to determine:\n  * Max volumes per pod\n  * Max volumes per node\n  * Max volumes per cluster\n* Test with the following volume types:\n  * EmptyDir\n  * Secret\n  * Configmap\n  * Downward API\n  * CSI mock driver\n  * TODO: Hostpath?\n  * TODO: Local?\n* Provide tests that vendors can easily run against their volume drivers.\n\n### Non-Goals\n\n* Test and measure storage provider’s drivers.\n\n## Proposal\n\n### Test Framework\n\nThe tests should be developed and run in sig-scalability’s test\n[Cluster Loader framework](https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/README.md)\nand infrastructure.\n\nThe framework already supports measurements for Pod startup latency and can be\nused to determine a [Pod startup SLO with\nvolumes](https://github.com/kubernetes/community/pull/3242). Any changes\nnecessary to this measurement will be handled by sig-scalability.\n\nAdditional measurements can be added to the framework to get informational details for\nfor volume operations. These are not strictly required but are nice to have:\n\n* Gather existing [volume operation\n  metrics](https://github.com/kubernetes/kubernetes/blob/master/pkg/volume/util/metrics.go)\n  from kube-controller-manager and kubelet.\n  * Caveat: These metrics measure the time of a single operation attempt and not\n    the e2e time across retries. These metrics may not be suitable to measure\n    latency from a pod's perspective.\n* Enhance the scheduler metrics collection to include\n  [volume scheduling\n  metrics](https://github.com/kubernetes/kubernetes/blob/master/pkg/controller/volume/persistentvolume/scheduler_bind_cache_metrics.go)\n* Timing of volume events. However, events should not be considered a reliable\n  measurement because they can be dropped or garbage collected. Also there are\n  only events for provision/delete, attach/detach. Mount events were removed due\n  to the high frequency of remount operations for API volume types (secrets,\n  etc). These are still useful to collect for debugging.\n\nEnhancements to Kubernetes itself can also be considered not just for scale\ntesting but for general improvements to supportability and debuggability of the\nsystem. These are not strictly required and can be considered as a future\nenhancement:\n\n* Add e2e metrics for volume operations that can track the time across operation\n  retries.\n* Add timestamps to the appropriate API object status whenever an operation is\n  completed.\n\n\n### Test Rollout\n\nBefore adding these tests to regular CI jobs, a performance baseline needs to be\nestablished to form the basis of a SLO. To accomplish this:\n\n1. Test parameters such as number of pods and number of volumes per pod should be\n  configurable.\n1. Start with the documented [max limits](https://kubernetes.io/docs/setup/cluster-large/)\n  and [stateless pod\n  SLO](https://github.com/kubernetes/community/blob/master/sig-scalability/slos/pod_startup_latency.md)\n  as a target SLO.\n1. Run the new scale tests with the same test parameters as the stateless pod density\n  tests.\n1. Adjust test parameters and/or target SLO as needed.\n1. Repeat until a stable target SLO can be established consistently across runs.\n\n### Test Portability\n\nIn order to make it easier for vendors to run these tests against their volume\ndrivers, the tests for persistent volumes should only use the default\nStorageClass configured for the cluster.\n\nTherefore these tests have the following prerequisites:\n\n* A Kubernetes cluster is brought up in an appropriate environment that can\n  support the storage provider.\n* The storage backend and driver is already installed and configured.\n* A default StorageClass for that driver been installed in the cluster.\n\n### Test Cases\n\n#### Pod Startup\n\nThese tests should measure how long it takes to start up a pod with unique volumes,\nassuming that the volumes have already been provisioned and volumes are not\nshared between pods.\n\nThe requirement for non-shared pods is so that we can produce an initial\nconsistent baseline measurement. The performance of pod startup latency when using\nshared volumes has a lot of variable factors such as volume type (RWO vs RWX),\nand whether or not the scheduler decided to schedule a replacement pod on the\nsame or different node. We can consider adding more test cases in the future\nthat can handle this scenario once we establish an initial baseline.\n\nFor the initial baseline, the test case can be run for each volume type by changing\nvarious dimensions (X):\n\n* Create 1 pod with X volumes and 1 node. This can help determine a limit for\n  max number of volumes in a single Pod.\n* Create X pods with 1 volume each on 1 node in parallel. This can help determine a limit\n  for max number of volumes on a single node.\n* Create X pods with 1 volume each in parallel.\n\nAll the test cases should measure pod startup time. A breakdown of time spent in volume scheduling,\nattach and mount operations is not strictly required but nice to have.\n\n\n### WIP Future Test Cases\n\nThese test cases are still under development and need to be refined before they\ncan be implemented in the future.\n\n#### Pod Teardown\n\nThese tests should measure how long it takes to delete a pod with volumes.\n\nFor each volume type:\n\n* Delete many pods with 1 volume each in parallel.\n* Delete 1 pod with many volumes.\n  * Measure pod deletion time, with a breakdown of time spent in unmount and detach operations.\n    * Note: Detach can only be measured for CSI volumes by the removal of the VolumeAttachment object.\n\n#### PV Binding Tests\n\nThese tests should measure the time it takes to bind a PVC to a preprovisioned available PV.\n\n#### PV Provisioning Tests\n\nThese tests should measure the time it takes to bind a PVC to a dynamically provisioned PV.\n\nFor each volume type that supports provisioning:\n\n* Create many PVCs with immediate binding in parallel.\n* Create many PVCs with delayed binding in parallel.\n  * Measure volume provisioning time.\n\n#### PV Deletion Tests\n\nThese tests should measure the time it takes to delete a PVC and PV.\n\nFor each volume type that supports deletion:\n\n* Delete many PVCs with the Delete reclaim policy in parallel.\n  * Measure volume deletion time.\n\n\n## Graduation Criteria\n\n### Phase 1\n\n* Pod startup tests running in scale clusters for all the targeted volume types.\n* Pod startup latency and max limits results published.\n\n### Phase 2\n\n* Improve pod startup latency measurements to get finer-grained volume operation\n  latencies.\n* Establish SLO for pod startup latency with volumes.\n* Tests fail if results exceeds SLO.\n\n### Phase 3\n\n* Add additional tests for Pod deletion and volume binding/provisioning.\n\n## Implementation History\n\n\n"
+  },
+  {
+    "id": "5c77f85e4d05924a5501963f14d9aa27",
+    "title": "KEP Template",
+    "authors": ["@jsafrane", "@gnufied"],
+    "owningSig": "sig-storage",
+    "participatingSigs": ["sig-scheduling"],
+    "reviewers": ["@bsalamat", "@thockin", "@msau42"],
+    "approvers": ["@bsalamat", "@msau42", "@thockin"],
+    "editor": "TBD",
+    "creationDate": "2019-04-08",
+    "lastUpdated": "2019-04-08",
+    "status": "implemented",
+    "seeAlso": [
+      "https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/20190129-csi-migration.md"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Volume Scheduling Limits\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n  - [Release Signoff Checklist](#release-signoff-checklist)\n  - [Summary](#summary)\n  - [Motivation](#motivation)\n    - [Goals](#goals)\n    - [Non-Goals](#non-goals)\n  - [Proposal](#proposal)\n    - [API Change](#api-change)\n    - [Implementation](#implementation)\n      - [Implementation Detail for all CSI Drivers](#implementation-detail-for-all-csi-drivers)\n      - [Implementation detail for in-tree Drivers with CSI migration disabled](#implementation-detail-for-in-tree-drivers-with-csi-migration-disabled)\n        - [When no CSI driver for same underlying storage type is installed on the node.](#when-no-csi-driver-for-same-underlying-storage-type-is-installed-on-the-node)\n        - [When CSI driver for same underlying storage type is installed on the node.](#when-csi-driver-for-same-underlying-storage-type-is-installed-on-the-node)\n      - [Implementation detail for in-tree Drivers with CSI migration enabled](#implementation-detail-for-in-tree-drivers-with-csi-migration-enabled)\n    - [User Stories](#user-stories)\n    - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n    - [Risks and Mitigations](#risks-and-mitigations)\n  - [Design Details](#design-details)\n    - [Test Plan](#test-plan)\n    - [Graduation Criteria](#graduation-criteria)\n    - [Upgrade / Downgrade / Version Skew Strategy](#upgrade--downgrade--version-skew-strategy)\n      - [Interaction with old \u003ccode\u003eAttachVolumeLimit\u003c/code\u003e implementation](#interaction-with-old--implementation)\n  - [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n## Summary\n\nNumber of volumes of certain type that can be attached to a node should be configurable easily and should be based on node type. This proposal implements dynamic attachable volume limits on a per-node basis rather than cluster global defaults that exist today. This proposal also implements a way of configuring volume limits for CSI volumes.\n\nThis proposal replaces [#730](https://github.com/kubernetes/enhancements/pull/730) and integrates volume limits for in-tree volumes (AWS EBS, GCE PD, AZURE DD, OpenStack Cinder) and CSI into one predicate. As result, in-tree volumes and corresponding CSI driver can share the same volume limit.\n\n## Motivation\n\nCurrent scheduler predicates for scheduling of pods with volumes is based on `node.status.capacity` and `node.status.allocatable`. It works well for hardcoded predicates for volume limits on AWS (`MaxEBSVolumeCount`), GCE(`MaxGCEPDVolumeCount`), Azure (`MaxAzureDiskVolumeCount`) and OpenStack (`MaxCinderVolumeCount`).\n\nIt is problematic for CSI (`MaxCSIVolumeCountPred`) outlined in [#730](https://github.com/kubernetes/enhancements/pull/730)\n\n- `ResourceName` is limited to 63 characters. We must prefix `ResourceName` with unique string (such as `attachable-volumes-csi-\u003cdriver name\u003e`) so it cannot collide with existing resources like `cpu` or `memory`. But `\u003cdriver name\u003e` itself is up to 63 character long, so we ended up with using SHA-sums of driver name to keep the `ResourceName` unique, which is not user readable.\n- CSI driver cannot share its limits with in-tree volume plugin e.g. when running pods with AWS EBS in-tree volumes and `ebs.csi.aws.com` CSI driver on the same node.\n\n### Goals\n\n- When CSI Driver is installed on the node, for in-tree drivers which are being considered for migration to CSI - same predicate will be used to handle Volume limit counting for in-tree as well as CSI Volumes. Similarly same limit will be used when user is using CSI or in-tree volumes on the node.\n\n- Existing predicates for in-tree volumes `MaxEBSVolumeCount`, `MaxGCEPDVolumeCount`, `MaxAzureDiskVolumeCount` and `MaxCinderVolumeCount`(now deprecated) will be removed when in-tree to CSI migration is GA and enabled by default for that particular volume plugin.\n  - When both deprecated in-tree predicate and CSI predicate are enabled, only `MaxCSIVolumeCountPred` does useful work and the other is NOOP to save CPU. This requires CSI Driver to be installed on the node.\n\n- Scheduler does not put pods that require CSI volumes to nodes that don't have the CSI driver installed.\n\n- Scheduler does not increase its CPU consumption. Any regression must be approved by sig-scheduling.\n  - Scheduler benchmark must be extended to schedule pods with volumes as part of this KEP.\n\nNote: Although we are saying existing predicates will become `NOOP` in this section and elsewhere, existing predicates still have to look up `CSINode`\nobject and return early as applicable.\n\n### Non-Goals\n\n- Heterogenous clusters, i.e. clusters where access to storage is limited only to some nodes. Existing `PV.spec.nodeAffinity` handling, not modified by this KEP, will filter out nodes that don't have access to the storage, so predicates changed in this KEP don't need to worry about storage topology and can be simpler.\n\n- Multiple plugins sharing the same volume limits. We expect that every CSI driver will have its own limits, not shared with other CSI drivers. In this KEP we support only in-tree volume plugins sharing its limits with one hard-coded CSI driver each.\n\n- Multiple \"units\" per single volume. Each volume used on a node takes exactly 1 unit from `allocatable.volumes`, regardless of the volume size, its replica count, number of connections to remote servers or other underlying resources needed to use the volume. For example, multipath iSCSI volume with three paths (and thus three iSCSI connections to three different servers) still takes 1 unit from `CSINode.spec.drivers[xyz].allocatable.volumes`.\n\n- Maximum capacity per node. Some cloud environments limit both number of attached volumes (covered in this KEP) and total capacity of attached volumes (not covered in this KEP). For example, this KEP will ensure that scheduler puts max. 128 GCE PD volumes to a [typical GCE node](https://cloud.google.com/compute/docs/machine-types#predefined_machine_types), but it won't ensure that the total capacity of the volumes is less than 64 TB.\n\n- Volume limits does not yet integrate with cluster autoscaler if all nodes in the cluster are running at maximum volume limits.\n\n## Proposal\n\nTrack volume limits for CSI driver in `CSINode` object instead of `Node` and update scheduler to use `CSINode` object to determining volume limits and availability of CSI driver.\n\nLimit in `CSINode` is used instead of limit coming from `Node` object whenever available for same in-tree volume type. This mean scheduler will always try to translate in-tree driver name to CSI driver name whenever `CSINode` object has same in-tree volume type (even if migration is off).\n\n  * To get rid of prefix + SHA for `ResourceName` of CSI volumes.\n  * So in-tree volume plugin can share limits with CSI driver that uses the same storage backend.\n\n### API Change\n\nCSINode is split into `spec` and `status`. `spec` contains list of drivers installed to the node and their properties that do not change during lifetime of a driver. `status` is missing right now, but it may be used later e.g. for driver health that changes in time.\n\nWe expect that limits of a CSI driver does not change during lifetime of a driver and therefore we put the resource limits into `CSINodeSpec`. The only way for a driver to change the limits is to deregister and register again, e.g. by restarting its container.\n\n```go\n// Until further notice, this is existing API to introduce full context.\n\ntype CSINode struct {\n\t...\n\n\t// spec is the specification of CSINode\n\tSpec CSINodeSpec `json:\"spec\" protobuf:\"bytes,2,opt,name=spec\"`}\n}\n\n// CSINodeSpec holds information about the specification of all CSI drivers installed on a node\ntype CSINodeSpec struct {\n\t// drivers is a list of information of all CSI Drivers existing on a node.\n\t// If all drivers in the list are uninstalled, this can become empty.\n\t// +patchMergeKey=name\n\t// +patchStrategy=merge\n\tDrivers []CSINodeDriver `json:\"drivers\" patchStrategy:\"merge\" patchMergeKey:\"name\" protobuf:\"bytes,1,rep,name=drivers\"`\n}\n\n// CSINodeDriver holds information about the specification of one CSI driver installed on a node\ntype CSINodeDriver struct {\n    // ...\n\n    // NEW API STARTS HERE\n    // Allocatable represents the resources of a node that are available for scheduling for volumes of this driver.\n    Allocatable VolumeLimits\n}\n\n// VolumeLimits is a set of resource limits for scheduling of volumes.\ntype VolumeLimits struct {\n\t// Count is maximum number of volumes provided by the CSI driver that can be used by the node\n\t// \"nil\" represents no limits - the node can handle any number of volumes of the driver.\n\tCount *int32 `json:\"count,omitempty\" protobuf:\"varint,1,opt,name=count`\n\n\t// Future proof: max. total size of volumes on the node can be added later\n}\n```\n\nCSINode example:\n\n```yaml\napiVersion: storage.k8s.io/v1beta1\nkind: CSINode\nmetadata:\n  name: ip-172-18-4-112.ec2.internal\nspec:\n  drivers:\n    - name: ebs.csi.aws.com\n      # Already existing fields\n      nodeID: ip-172-18-4-112.ec2.internal\n      topologyKeys:\n         # ...\n      # New API:\n      allocatable:\n        # AWS node can attach max. 40 volumes, 1 is reserved for the system\n        count: 39\n    - name: org.kernel.nfs\n      allocatable:\n        # NFS does not impose any limits of volumes mounted to the node\n        count:    # nil means \"no limit\"\n```\n\n### Implementation\n\nSection below describes behaviour of old predicates, CSI predicate and scheduler after the proposal has been implemented.\nFor brevity - \"old predicates\" refers to now deprecated cloudprovider specific predicates - `MaxEBSVolumeCount`, `MaxGCEPDVolumeCount`, `MaxAzureDiskVolumeCount` and `MaxCinderVolumeCount`.\n\n#### Implementation Detail for all CSI Drivers\n\n* Kubelet will create `CSINode` instance during initial CSI Driver registration.\n  * Limits of each CSI volume plugin will be added to `CSINode.spec.drivers[xyz].allocatable`.\n  * User may NOT change `CSINode.spec.drivers[xyz].allocatable` to override volume plugin / CSI driver values, e.g. to \"reserve\" some attachment to the operating system. Kubelet will periodically reconcile `CSINode` and overwrite the value.\n    * Especially, `kubelet --kube-reserved` or `--system-reserved` cannot be used to \"reserve\" volumes for kubelet or the OS. It is not possible with existing kubelet and this KEP does not change it. We expect that CSI drivers will have configuration options / cmdline arguments to reserve some volumes and they will report their limit already reduced by that reserved amount.\n* Scheduler will respect `Node.status.allocatable` and `Node.status.capacity` for CSI volumes if `CSINode` object is not available or has missing entry in `CSINode.spec.drivers[xyz].allocatable` during a deprecation period but kubelet will stop populating `Node.status.allocatable` and `Node.status.capacity` for CSI volumes.\n  * After deprecation period for CSI volumes, limits coming from `Node.status.allocatable` and `Node.status.capacity` will be completely ignored by the scheduler.\n* Scheduler won't schedule pods with volumes (in-tree or CSI) for which migration has been enabled and driver is not installed on the node yet.\n  * Volumes for which there is no in-tree to CSI migration plan will follow a deprecation cycle before.\n  * Important: this can only be implemented once volume limits are integrated with the cluster autoscaler.\n\n#### Implementation detail for in-tree Drivers with CSI migration disabled\n\n##### When no CSI driver for same underlying storage type is installed on the node.\n\n* For Azure, GCEPD, AWS and Cinder - in-tree volume plugins will keep reporting their limits via `Node` object and old predicates will work as expected until CSI migration has been enabled (and GA) for given volume plugin.\n\n##### When CSI driver for same underlying storage type is installed on the node.\n\n* For Azure, GCEPD, AWS and Cinder - in-tree volume plugins will report their limits via `Node` object same as before.\n\n#### Implementation detail for in-tree Drivers with CSI migration enabled\n\n* For Azure, GCEPD, AWS and Cinder - in-tree volume plugins will report their limits via `Node` object same as before.\n* Old predicates will be modified to perform an additional check of `CSINode`. If they detect that the CSI migration has been enabled for the volume, the old predicate will return early (with success) and `MaxCSIVolumeCountPred` will be responsible for counting both CSI and in-tree volumes of same type.\n\n### User Stories\n\n### Implementation Details/Notes/Constraints\n\n[CSI migration library](https://github.com/kubernetes/kubernetes/tree/master/staging/src/k8s.io/csi-translation-lib) is used to find CSI driver name for in-tree volume plugins + its `VolumeHandle`. This CSI driver name is used as key in `CSINode.CSINode.spec.drivers[xyz].allocatable` list. The `VolumeHandle` is unique for each volume and will be used to de-duplicate volumes used by multiple pods on the same node.\n\n### Risks and Mitigations\n\n* This KEP depends on [CSI migration library](https://github.com/kubernetes/kubernetes/tree/master/staging/src/k8s.io/csi-translation-lib). It can happen that CSI migration is redesigned / cancelled.\n  * Countermeasure: [CSI migration](https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/20190129-csi-migration.md) and this KEP should graduate together.\n\n* This KEP depends on [CSI migration library](https://github.com/kubernetes/kubernetes/tree/master/staging/src/k8s.io/csi-translation-lib) ability to handle in-line in-tree volumes. Scheduler will need to get CSI driver name + `VolumeHandle` from them to count them towards the limit.\n\n## Design Details\n\nExisting feature gate `AttachVolumeLimit` will be re-used for implementation of this KEP. The feature is already beta and is enabled by default.\n\n### Test Plan\n\n* [Scheduler benchmark](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-scheduling/scheduler_benchmarking.md) must be extended to run pods with volumes as part of this KEP. Following matrix will be tested:\n  * Predicates:\n    * All volume predicates enabled.\n    * Only deprecated `MaxEBSVolumeCount`, `MaxGCEPDVolumeCount`, `MaxAzureDiskVolumeCount` and `MaxCinderVolumeCount` predicates enabled.\n    * Only `MaxCSIVolumeCountPred` predicate enabled.\n  * API objects:\n    * Both CSINode and Node containing `spec/status.allocatable` for a volume plugin (to simulate kubelet during deprecation period).\n    * Only CSINode containing `spec.drivers[xyz].allocatable` for a volume plugin (to simulate kubelet after deprecation period).\n    * Only Node containing `status.allocatable` for a volume plugin (to simulate old kubelet).\n  * Test results should be ideally the same as before the KEP.\n    * Any deviation needs to be approved by sig-scheduling.\n\n* Run e2e tests and kubelet version skew tests to check that scheduler picks the right values from CSINode or Node.\n\n* Add e2e test that runs pods with both in-tree volumes and CSI driver for the same storage backend and check that they share the same volume limits.\n\n### Graduation Criteria\n\n##### Alpha -\u003e Beta Graduation\n\nN/A (`AttachVolumeLimit` feature is already beta).\n\n##### Beta -\u003e GA Graduation\n\nIt must graduate together with CSI migration. We can enable caching of in-use volumes on a node to improve performance before going GA.\n\n### Upgrade / Downgrade / Version Skew Strategy\n\nDuring upgrade, downgrade or version skew, kubelet may be older that scheduler. Kubelet will not fill `CSINode.spec` with volume limits and it will fill volume limits into `Node.status`. Scheduler must fall back to `Node.status` when `CSINode` is not available or its `spec` does not contain a volume plugin / CSI driver.\n\n#### Interaction with old `AttachVolumeLimit` implementation\n\nDue to version skew, following situations are possible (scheduler is always with `AttachVolumeLimit` enabled and with this KEP implemented):\n\n* Kubelet has `AttachVolumeLimit` off:\n  * Scheduler does not see any volume limits in `CSINode` nor `Node`.\n  * In-tree volumes: since `CSINode` is missing, scheduler falls back to `MaxEBSVolumeCount`, `MaxGCEPDVolumeCount`, `MaxAzureDiskVolumeCount` and `MaxCinderVolumeCount` predicates and schedules in-tree volumes the old way with hardcoded limits.\n  * CSI: from scheduler point of view, the node can handle any number of CSI volumes.\n\n* Kubelet has old implementation of `AttachVolumeLimit` and the feature is on (kubelet fills `Node.status.available`):\n  * Scheduler does not see any volume limits in `CSINode`.\n  * In-tree: Since `CSINode` is missing, scheduler falls back to `MaxEBSVolumeCount`, `MaxGCEPDVolumeCount`, `MaxAzureDiskVolumeCount` and `MaxCinderVolumeCount` predicates and schedules in-tree volumes the old way.\n  * CSI: Scheduler falls back to told implementation of `MaxCSIVolumeCountPred` for CSI volumes and uses limits from `Node.status`.\n\n* Kubelet has new implementation of `AttachVolumeLimit` and the feature is on (kubelet fills `CSINode`):\n  * No issue here, see this KEP.\n  * Since `CSINode` is available, scheduler uses new implementation of `MaxCSIVolumeCountPred`.\n\nAs implied by the above, the scheduler needs to have both old and new implementation of `MaxCSIVolumeCountPred` and switch between them based on `CSINode` availability for a particular node until the old implementation is deprecated and removed (2 releases).\n\n## Implementation History\n\n* K8s 1.11: Alpha\n* K8s 1.12: Beta\n* K8s 1.17: GA\n\n# Alternatives\n\nIn https://github.com/kubernetes/enhancements/pull/730 we tried to merge volume limits in `Node.status.capacity` and `Node.status.attachable`. We discovered these issues:\n\n* We cannot use plain CSI driver name as resource name `Node.status.attachable`, as it could collide with other resources (e.g. \"memory\"), so we added volume specific prefix.\n* Since CSI driver name can be [up to 63 character long](https://github.com/container-storage-interface/spec/blob/master/spec.md#getplugininfo), the prefix + driver name it cannot fit 64 character resource name limit. We ended up hashing the driver name to save space.\n\nBy moving volume limit to CSINode we fix both issues.\n"
+  },
+  {
+    "id": "914a216f3277f15db3eb297162a75907",
+    "title": "CSI Snapshot",
+    "authors": ["@jingxu97", "@xing-yang"],
+    "owningSig": "sig-storage",
+    "participatingSigs": ["sig-storage"],
+    "reviewers": ["@msau42", "@saad-ali", "@thockin"],
+    "approvers": ["@msau42", "@saad-ali", "@thockin"],
+    "editor": "TBD",
+    "creationDate": "2019-07-09",
+    "lastUpdated": "2019-07-29",
+    "status": "implementable",
+    "seeAlso": ["n/a"],
+    "replaces": ["n/a"],
+    "supersededBy": ["n/a"],
+    "markdown": "\n# Title\n\nCSI Snapshot\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Test Plan](#test-plan)\n  - [Unit tests](#unit-tests)\n  - [E2E tests](#e2e-tests)\n- [Graduation Criteria](#graduation-criteria)\n  - [Alpha-\u0026gt;Beta](#alpha-beta)\n  - [Beta-\u0026gt;GA](#beta-ga)\n- [Changes](#changes)\n  - [Changes Implemented](#changes-implemented)\n  - [Work in Progress](#work-in-progress)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis KEP is written after the original design doc has been approved and implemented. Design for CSI Volume Snapshot Support in Kubernetes is incorporated as part of the [CSI Volume Snapshot in Kubernetes Design Doc](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/csi-snapshot.md).\n\nThe rest of the document includes required information missing from the original design document: test plan and graduation criteria.\n\n## Test Plan\n\n### Unit tests\n\n* Unit tests around snapshot creation and deletion logic.\n* Unit tests around VolumeSnapshot and VolumeSnapshotContent binding logic.\n* Unit tests for creating volume from snapshot.\n\n### E2E tests\n\n* (P0) e2e tests for creating/deleting snapshot.\n* (P0) e2e tests for creating volume from snapshot.\n* (P1) e2e tests for delete/retain policy.\n* (P1) e2e tests for deleting API objects out of order (snapshot protection).\n* (P2) e2e tests for secret fields.\n* (P2) e2e tests for metrics.\n\n## Graduation Criteria\n\n### Alpha-\u003eBeta\n\n* Feature complete, including:\n  * Create/delete volume snapshots\n  * Create new volumes from a snapshot\n  * SnapshotContent Deletion/Retain Policy\n  * Snapshot Object in Use Protection\n  * Separate the common controller from the sidecar controller\n  * Add secrets field to list-snapshots RPC in the CSI spec. Add “data-source-secret” in create-volume intended for accessing the data source. Implement them in external-snapshotter and external-provisioner.\n  * Add metrics support\n* Unit and e2e tests implemented\n* Update snapshot CRDs to v1beta1 and enable VolumeSnapshotDataSource feature gate by default.\n\n### Beta-\u003eGA\n\n* Snapshot feature is used as a basic building block in other advanced applications. \n* Feature deployed in production and have gone through at least one K8s upgrade.\n\n## Changes\n\n### Changes Implemented\n\nHere are the changes since the original design proposal:\n\n* Renamed `Ready` to `ReadyToUse` in the `Status` field of `VolumeSnapshot` API object.\n* Changed type of `RestoreSize` in `CSIVolumeSnapshotSource` from `*resource.Quantity`  to `*int64`.\n* Lease based Leader Election support is added.\n* Added `VolumeSnapshotContent` deletion policy which is also specified in `VolumeSnapshotClass`.\n* Added `VolumeSnapshot` and `VolumeSnapshotContent` in Use Protection using Finalizers.\n* Added Finalizer on the snapshot source PVC to prevent it from being deleted when a snapshot is being created from it.\n* Added check to see whether ListSnapshots is supported by the CSI driver. If it is supported, ListSnapshots will be called to find out the status of a snapshot during static binding; otherwise it is assumed the snapshot ID provided by the admin is valid.\n\n### Work in Progress\n\nThere are other things we are working on for Beta:\n\n* If snapshot creation times out, VolumeSnapshot status will not be marked as failed so that controller will continue to retry to create until the operation either succeeds or fails. It is up to the user or an upper level application that uses the VolumeSnapshot to determine what to do with the snapshot. This work is on-going.\n* Investigation is on-going to determine whether we should separate common controller logic from other logic that belongs to the sidecar. Can be in the same external-snapshotter repo. The common controller should not be deployed with the driver. It should be deployed by the cluster deployer, or we can provide a way to deploy it as a separate Statefulset, not together with the driver. CRD installation should also be done by the cluster deployer.\n* Add secrets field to list-snapshots RPC in the CSI spec. Add “data-source-secret” in create-volume intended for accessing the data source. Implement them in external-snapshotter and external-provisioner.\n* Add metrics support in the snapshot controller.\n  * operational end to end latency metrics.\n    labels:\n    * operation_name, i.e., creation-snapshot, delete-snapshot\n    * csi-driver-name\n  * operation error count.\n    labels:\n    * operation_name, i.e., creation-snapshot, delete-snapshot\n    * csi-driver-name\n* Update snapshot CRDs to v1beta1 and enable VolumeSnapshotDataSource feature gate by default.\n\n## Implementation History\n\nVolume snapshot is implemented as alpha feature in this repo in Kubernetes v1.12:\nhttps://github.com/kubernetes-csi/external-snapshotter\n\nFeature gate is added by this PR:\nhttps://github.com/kubernetes/kubernetes/pull/67087\n"
+  },
+  {
+    "id": "bdec5398a6549d8f91e2b07e599d2259",
+    "title": "Raw Block Volumes",
+    "authors": ["@jsafrane"],
+    "owningSig": "sig-storage",
+    "participatingSigs": ["sig-storage"],
+    "reviewers": ["@msau42", "@saad-ali"],
+    "approvers": ["@saad-ali"],
+    "editor": "TBD",
+    "creationDate": "2019-10-08",
+    "lastUpdated": "2019-15-10",
+    "status": "implementable",
+    "seeAlso": [
+      "https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/raw-block-pv.md"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Raw Block Volumes\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n- [Upgrade/Downgrade Strategy](#upgradedowngrade-strategy)\n- [Test Plan](#test-plan)\n  - [Unit tests](#unit-tests)\n  - [E2E tests](#e2e-tests)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n  - [K8s 1.9: Alpha](#k8s-19-alpha)\n  - [K8s 1.13: Beta](#k8s-113-beta)\n  - [K8s 1.17: GA](#k8s-117-ga)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis document presents a proposal for managing raw block storage in Kubernetes\nusing the persistent volume source API as a consistent model of consumption.\n\nNote that is has been designed \u0026 merged before KEP process was introduced:\nhttps://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/raw-block-pv.md\n\n## Motivation\n\nBy extending the API for volumes to specifically request a raw block device,\nwe provide an explicit method for volume consumption, whereas previously any\nrequest for storage was always fulfilled with a formatted filesystem, even when\nthe underlying storage was block. In addition, the ability to use a raw block\ndevice without a filesystem will allow Kubernetes better support of high\nperformance applications that can utilize raw block devices directly for their\nstorage. Block volumes are critical to applications like databases (MongoDB,\nCassandra) that require consistent I/O performance and low latency. For mission\ncritical applications, like SAP, block storage is a requirement.\n\n### Goals\n\n* Enable durable access to block storage\n* Provide flexibility for users/vendors to utilize various types of storage devices\n* Agree on API changes for block\n* Provide a consistent security model for block devices \n* Provide a means for running containerized block storage offerings as non-privileged container\n\n### Non-Goals\n\n* Support all storage devices natively in upstream Kubernetes. Non-standard storage devices are expected to be managed using extension\n  mechanisms.\n* Provide a means for full integration into the scheduler based on non-storage related requests (CPU, etc.)\n* Provide a means of ensuring specific topology to ensure co-location of the data \n* CSI volume plugin changes - CSI block volumes are tracked as a separate KEP.\n\n## Proposal\n\nSee original design proposal at\nhttps://raw.githubusercontent.com/kubernetes/community/master/contributors/design-proposals/storage/raw-block-pv.md\n\n## Upgrade/Downgrade Strategy\n\nThese situations can happen when various Kubernetes components run with raw block volume feature gate on/off:\n\n* API server on, controller-manager on, kubelet off:\n  * When processing new pods in VolumeManager, it checks that a filesystem PV\n    is used only in `volumeMounts` and block PV in `volumeDevices` and\n    rejects any mismatched PV/Pod to protect a block PV from being formatted\n    and mounted.\n    For that, it evaluates appropriate block-related fields in Pod, PV and PVC\n    even when the feature gate is disabled.\n  * In all other cases, kubelet does not see `volumeDevices` section in pods\n    and thus it will run the pods as if the pods did not use the volume at all.\n    Kubelet will not touch block volumes, especially it will not format / mount /\n    resize them. Especially, when the feature is disabled in kubelet while\n    a pod is running and uses a block volume, the volume may not be cleaned\n    up properly when the pod is deleted. Manual cleanup may be necessary.\n\n* API server on, controller-manager off:\n  * PV controller will not bind block PV to a filesystem PVC and filesystem PV\n    to block PVC. Such PVC cannot be used by the cluster in any pod.\n    For that, it evaluates appropriate block-related fields in PV and PVC\n    even when the feature gate is disabled.\n\n* API server off: all newly created PVs / PVCs / Pods that refer to a block\n  volume are rejected by validation. Older objects may keep the field to\n  prevent from data corruption (see previous bullets).\n\nAs result, cluster admins are responsible for deleting any pods that use block\nvolumes before downgrading to an older release / disabling the feature gate.\n\n## Test Plan\n\n### Unit tests\n\n* Kubelet VolumeManager can provide a block volume using a (fake) volume plugin\n  to (fake) container runtime.\n* PV controller can bind block PVs to block PVCs.\n* PV controller can provision block PVs to block PVCs.\n\n### E2E tests\n\n* Implement the same e2e tests as we have for filesystem volumes.\n\n## Graduation Criteria\n\n### Alpha -\u003e Beta\nAlready happened.\n\n### Beta -\u003e GA\n* Implement missing e2e tests:\n  * Mismatched usage of filesystem / block volumes (#79796).\n  * Block volume reconstruction after kubelet restart (#83451).\n* Implement reconstruction of local volumes.\n* Manual stress test with at least block volume plugin is performed with a node\n  running non-trivial amount of pods that use a block device (simple `dd`\n  should do).\n  * Test that the pods are moved to another nodes and data is retained when\n    the node is drained.\n  * Test that the data is retained when kubelet restarts while the pods are\n    running.\n  * Test that the data is retained when the node reboots. For pods that were\n    moved to another nodes during the outage, test that the newly started node\n    cleaned up their devices.\n* Manual test with API server with enabled block volume feature gate and\n  kubelet with the gate disabled.\n\n## Implementation History\n\n### K8s 1.9: Alpha\n* Initial implementation.\n\n### K8s 1.13: Beta\n* Enhanced e2e tests.\n* Most block-based volume plugins implemented block volume plugin interfaces.\n\n### K8s 1.17: GA\n* Fixed block volume reconstruction.\n* Stress tests as noted above.\n"
+  },
+  {
+    "id": "a5dee463e07a56cc0b6c46a38191979e",
+    "title": "Immutable Secrets and ConfigMaps",
+    "authors": ["@wojtek-t"],
+    "owningSig": "sig-storage",
+    "participatingSigs": ["sig-apps", "sig-node", "sig-scalability"],
+    "reviewers": ["@yujuhong", "@lavalamp", "@msau42"],
+    "approvers": ["@saad-ali"],
+    "editor": "",
+    "creationDate": "2019-11-17",
+    "lastUpdated": "2019-12-09",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Immutable ephemeral volumes\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n  - [Define immutability at VolumeSource level](#define-immutability-at-volumesource-level)\n  - [Optimize watches](#optimize-watches)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n**ACTION REQUIRED:** In order to merge code into a release, there must be an issue in [kubernetes/enhancements] referencing this KEP and targeting a release milestone **before [Enhancement Freeze](https://github.com/kubernetes/sig-release/tree/master/releases)\nof the targeted release**.\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e., [kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be updated for the enhancement to be released.\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nThe most popular and the most convinient way of consuming Secrets and\nConfigMaps by Pods is consuming it as a file. However, any update to\na Secret or ConfigMap object is quickly (roughly within a minute)\nreflected in updates of the file mounted for *all* Pods consuming them.\nThat means, that a bad update (push) of Secret and/or ConfigMap can\nvery quickly break the whole appplication.\n\nThe recommended approach for upgrading applications are obviously\nrolling upgrades. For Secrets/ConfigMaps this is translating to\ncreating a new object and updating PodTemplate with the reference\nto it. However, that doesn't protect users from outages caused by\naccidental bad updates of existing Secrets and/or ConfigMaps.\n\nMoreover, the feature of updating Secrets/ConfigMaps for running\nPods is also expensive from scalability/performance point of view.\nEvery Kubelet has to watch (default) or periodically poll every\nunique Secret and ConfigMap referenced by any of the Pods it is\nrunning.\n\n## Motivation\n\nIn this KEP, we are proposing to introduce an ability to specify\nthat contents of a particular Secret/ConfigMap should be immutable\nfor its whole lifetime. For those Secrets/ConfigMap, Kubelets will\nnot be trying to watch/poll for changes to updated mounts for their\nPods.\nGiven there are a lot of users not really taking advantage of automatic\nupdates of Secrets/ConfigMaps due to consequences described above, this\nwill allow them to:\n- protect themselves better for accidental bad updates that could cause\n  outages of their applications\n- achieve better performance of their cluster thanks to significant\n  reduction of load on apiserver\n\n### Goals\n\n- introduce protection mechanism to avoid outages due to accidental\n  updates of existing Secrets/ConfigMaps\n- improve cluster performance by reducing load on Kubernetes control\n  plane (mostly kube-apiserver) consumed by a feature many people\n  would be willing to tradeoff for better scale/performance\n\n### Non-Goals\n\n- change the default behavior of consumption of Secrets/ConfigMaps\n\n## Proposal\n\nWe propose to extend `ConfigMap` and `Secret` types with an additional\nfield:\n```go\n  Immutable *bool\n```\n\nIf set, the machinery in apiserver will reject any updates of the object\ntrying to change anything different than ObjectMetadata.\n\nNote that will NOT reject all updates of the object, as we need to allow\ne.g. for mutating ObjectMetadata (to not break object lifecycle, e.g. by\nintroducing a deadlock if Finalizers are set) or to allow rotating\ncertificates used for encryption at rest. We will only reject requests\nthat are explicitly changing keys and/or values stored in Secrets and/or\nConfigMaps.\n\nBased on the value of `Immutable` field Kubelet will or will not:\n- start a watch (or periodic polling) of a given Secret/ConfigMap\n- perform updates of files mounted to a Pod based on updates of\n  the Kubernetes object\n\n### Risks and Mitigations\n\nGiven how closely the implementation of the feature will be related to\nthe implementation of automatic updates of Secrets/ConfigMaps, there is\na risk for introducing a bug and breaking that feature. The existing\nunit and e2e tests should catch that, but we will audit them and add\nnew ones to cover the gaps if needed. Additionally, we will try to hide\nthe new logic behind the feature gate.\n\n## Design Details\n\n### Test Plan\n\nFor `Alpha`, we will add e2e tests verifying that contents of Secrets and\nConfigMaps marked as immutable really can't be updated. Additionally, these\nwill check if the metadata can be modified.\n\nAdditionally, unit tests will be added in Kubelet codebase to ensure that\nthe newly added logic to not watch immutable Secrets/ConfigMaps works as\nexpected.\n\nFor `Beta`, we will also extend scalability tests with a number of immutable\n`Secrets` and `ConfigMaps` to validate the performance impact (for `Alpha`\nonly manual scalability tests will be performed).\n\n### Graduation Criteria\n\nAlpha:\n- All tests describe above for `Alpha` are implemented and passing.\n- Manual scalability tests prove the expected impact.\n\nBeta:\n- Scalability tests are extended to mount an immutable Secret and ConfigMap\nfor every single Pod, and that doesn't violate existing SLOs.\n\nGA:\n- No complaints about the API and user bug reports for 2 releases.\n\n### Upgrade / Downgrade Strategy\n\nNo upgrade/downgrade concerns.\n\n### Version Skew Strategy\n\nOn Nodes in versions on supporting the feature, Kubelet will still be watching\nimmutable Secrets and/or ConfigMaps. That said, this is purely a performance\nimprovement and doesn't have correctness implications. So those clusters will\nsimple have worse scalability characteristic.\n\n## Implementation History\n\n2019-11-18: KEP opened\n2019-12-09: KEP marked implementable\n\n## Alternatives\n\n### Define immutability at VolumeSource level\n\nInstead of making an object immutable, we could define immutability\nper mount in VolumeSource.\n\nPros:\n- higher granularity\nCons:\n- unclear/tricky semantic on Kubelet restarts and Pod restarts/updates\n\n### Optimize watches\n\nWe could potentially address scalability/performance aspect by optimizing\napimachinery. However, the bottlenecks seem to be mainly at the level of\nGolang memory allocations.\n\nPros:\n- no additional API\nCons:\n- doesn't protect against unexpected bad updates causing outages\n- unclear to what extent we can push the limits here (if at all)\n"
+  },
+  {
+    "id": "5336c1d8147494cc9c535bae5f992519",
+    "title": "New label for trusted PR identification",
+    "authors": ["@matthyx"],
+    "owningSig": "sig-testing",
+    "participatingSigs": ["sig-contributor-experience"],
+    "reviewers": ["@fejta", "@cjwagner", "@BenTheElder", "@cblecker", "@stevekuznetsov"],
+    "approvers": ["TBD"],
+    "editor": "TBD",
+    "creationDate": "2018-06-25",
+    "lastUpdated": "2019-01-30",
+    "status": "implemented",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# New label for trusted PR identification\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n  - [Benefits](#benefits)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Graduation Criteria](#graduation-criteria)\n- [Future evolutions](#future-evolutions)\n- [References](#references)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThis document describes a major change to the way the `trigger` plugin determines if test jobs should be started on a pull request (PR).\n\nWe propose introducing a new label named `ok-to-test` that will be applied on non-member PRs once they have been `/ok-to-test` by a legitimate reviewer.\n\n## Motivation\n\nPR test jobs are started by the trigger plugin on *trusted PR* events, or when a *untrusted* PR becomes *trusted*.\n\u003e A PR is considered trusted if the author is a member of the *trusted organization* for the repository or if such a member has left an `/ok-to-test` command on the PR.\n\nIt is easy spot an untrusted PR opened by a non-member of the organization by its `needs-ok-to-test` label. However the contrary is difficult and involves scanning every comment for a `/ok-to-test`, which increases code complexity and API token consumption.\n\n### Goals\n\nThis KEP will only target PRs authored from non-members of the organization:\n\n* introduce a new `ok-to-test` label\n* modify `/ok-to-test` command to apply `ok-to-test` on success\n* modify `trigger` plugin and other tools to use `ok-to-test` for PR trust\n* support `/ok-to-test` calls inside review comments\n\n### Non-Goals\n\nThis KEP will not change the current process for members of the organization.\n\n## Proposal\n\nWe suggest introducing a new label named `ok-to-test` that would be required on any non-member PR before automatic test jobs can be started by the `trigger` plugin.\n\nThis label will be added by members of the *trusted organization* for the repository using the `/ok-to-test` command, detected with a single GenericCommentEvent handler on corresponding events (issue_comment, pull_request_review, and pull_request_review_comment).\n\n### Implementation Details/Notes/Constraints\n\n1. PR: declare `ok-to-test`\n   * add `ok-to-test` to `label_sync`\n1. (custom tool needed) batch add `ok-to-test` label to non-members trusted PRs\n   * for all PR without `ok-to-test` or `needs-ok-to-test`\n   * if author is not a member of trusted org\n   * add `ok-to-test`\n1. documentation updates\n   * edit all documentation references to `needs-ok-to-test`\n1. other cookie crumbs updates\n   * update infra links (eg: redirect https://go.k8s.io/needs-ok-to-test)\n   * update locations where edits are expected (eg: https://github.com/search?q=org%3Akubernetes+needs-ok-to-test\u0026type=Code)\n1. PR: switch to `ok-to-test`\n   * remove `needs-ok-to-test` from `missingLabels` in `prow/config.yaml`\n   * edit `prow/config/jobs_test.go`\n   * edit `prow/cmd/deck/static/style.css`\n   * edit `prow/cmd/tide/README.md`\n   * code changes in `trigger`:\n      * `/ok-to-test` adds `ok-to-test`\n      * PR trust relies on `ok-to-test`\n      * if PR has both labels, drop `needs-ok-to-test`\n      * edit all references to `needs-ok-to-test`\n1. run batch job again, to catch new PRs that arrived between first run and merge/deploy\n1. (to be discussed) periodically check for and report PRs with both `ok-to-test` and `needs-ok-to-test` labels\n\n### Benefits\n\n* Trusted PRs are easily identified by either being authored by org members, or by having the `ok-to-test` label.\n* Race conditions can no longer happen when checking if a PR is trusted.\n* API tokens are saved by avoiding listing the comments, reviews, and review comments every time we need to check if a PR is trusted.\n\n### Risks and Mitigations\n\nN/A\n\n## Graduation Criteria\n\nN/A\n\n## Future evolutions\n\nIn the future, we might decide to require the new label for all PRs, which means that organization members will also need the `ok-to-test` label applied to their PRs before automatic testing can be triggered.\n\nTrusted and untrusted PRs will be even easier to tell apart.\n\nThis would require adding automatically the `ok-to-test` label to member authored PRs to keep the current functionality.\n\n## References\n\n* https://github.com/kubernetes/test-infra/issues/3827\n* https://github.com/kubernetes/test-infra/issues/7801\n* https://github.com/kubernetes/test-infra/pull/5246\n\n## Implementation History\n\n* 2018-06-25: creation of the KEP\n* 2018-07-09: KEP content LGTM during sig-testing presentation\n* 2018-07-24: KEP updated to keep `needs-ok-to-test` for better UX\n* 2018-09-03: KEP rewritten with template\n* 2018-10-04: KEP merged into master\n* 2018-10-08: start of implementation\n* 2018-10-10: `ok-to-test` label added\n* 2019-01-29: remove comment parsing code for PR trust\n"
+  },
+  {
+    "id": "ba543b893e8edd24d0bbb72045dfa690",
+    "title": "Breaking apart the Kubernetes test tarball",
+    "authors": ["@ixdy"],
+    "owningSig": "sig-testing",
+    "participatingSigs": ["sig-release"],
+    "reviewers": ["@akutz", "@amwat"],
+    "approvers": ["@spiffxp", "@tpepper"],
+    "editor": "TBD",
+    "creationDate": "2019-01-18",
+    "lastUpdated": "2019-03-06",
+    "status": "implemented",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Breaking apart the kubernetes test tarball\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Internal structure of the test tarball](#internal-structure-of-the-test-tarball)\n    - [Binary artifacts](#binary-artifacts)\n    - [Portable sources](#portable-sources)\n  - [Updating dependencies on \u003ccode\u003ekubernetes-test.tar.gz\u003c/code\u003e](#updating-dependencies-on-)\n    - [Dependencies outside the Kubernetes organization](#dependencies-outside-the-kubernetes-organization)\n  - [Risks and Mitigations](#risks-and-mitigations)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n- [References](#references)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [x] [k/enhancements issue in release milestone and linked to KEP](https://github.com/kubernetes/enhancements/issues/714)\n- [x] KEP approvers have set the KEP status to `implementable`\n- [x] Design details are appropriately documented\n- [x] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [x] Graduation criteria is in place\n- [x] \"Implementation History\" section is up-to-date for milestone\n- [x] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nThe Kubernetes release artifacts include a \"mondo\" test tarball which includes both\n\"portable\" test sources (such as shell scripts and image manifests) as well as\nplatform-specific test binaries for all supported client, node, and\nserver platforms.\n\nThis KEP proposes replacing the monolith test tarball with platform-specific\ntarballs, matching the existing pattern used for the client, node, and server\ntarballs.\n\n## Motivation\n\nAs the number of supported client, server, and node platforms has increased, the\nsize of the test tarball has increased as well. As of Kubernetes v1.13.2, the\nofficial `kubernetes-test.tar.gz` is approximately 1.2GB; previous releases\nranged from 1.3 - 1.5GB. Several years ago,\n[there were complaints](https://github.com/kubernetes/kubernetes/issues/28435)\nthat the \"full\" `kubernetes.tar.gz` release tarball was too big at 1.4G.\nMany of the motivations for breaking up that tarball echo into this proposal.\n\nThe Bazel effort is another driving motivation. It's possible to build all\nrelease artifacts solely using Bazel, and there is progress being made on\nsupporting cross-compilation of binary artifacts, but combining multiple\ntarget platforms in one Bazel call is currently not well-supported.\nSeparating this tarball would make it easier to ensure that we can use Bazel t\nproduce identical artifacts as the non-Bazel build.\n\n### Goals\n\n- The Kubernetes test artifacts are broken apart such that users only need to\n  download binaries for the platforms they're using.\n- Be largely invisible to most developers: everything should just keep working.\n\n### Non-Goals\n\n- Changing the underlying build system. Both the make/shell-based build\n  system and the Bazel-based build system will be affected, but users can\n  continue to use their existing workflow.\n- Removing cruft from the test tarballs. Likely, there are binaries and portable\n  sources no longer being used anywhere, but we won't prune them with this\n  effort.\n- Changing what is released independent of the test tarball; i.e. whether we\n  should make test binaries able to be downloaded directly from GCS.\n\n## Proposal\n\nInstead of building and distributing a single `kubernetes-test.tar.gz` with all\nportable sources and compiled binaries for all platforms, produce several\nplatform-specific tarballs, one for each platform defined in\n[`KUBE_TEST_PLATFORMS`](https://github.com/kubernetes/kubernetes/blob/193f659a1cd454b93cbe1e7b1f13b77c21783461/hack/lib/golang.sh#L150-L160):\n\n- `kubernetes-test-linux-amd64.tar.gz`\n- `kubernetes-test-linux-arm.tar.gz`\n- `kubernetes-test-linux-arm64.tar.gz`\n- `kubernetes-test-linux-s390x.tar.gz`\n- `kubernetes-test-linux-ppc64le.tar.gz`\n- `kubernetes-test-darwin-amd64.tar.gz`\n- `kubernetes-test-windows-amd64.tar.gz`\n\n### Internal structure of the test tarball\n\nAt present, the Kubernetes test tarball has several components, all\nrooted under a `kubernetes/` top-level directory.\n\n#### Binary artifacts\n\nThe test binary artifacts are currently organized into directories divided by platform:\n\n- `platforms/`\n  - `darwin/`, `linux/`, `windows/`\n    - `amd64/`, `arm/`, `arm64/`, `ppc64le/`, `s390x/`\n\nFor comparison, the existing platform-specific tarballs\n(such as `kubernetes-client-linux-amd64.tar.gz`) place all binaries under\na constant path with no platform information: `kubernetes/client/bin/kubectl`.\n\nScripts (such as `cluster/get-kube-binaries.sh`) [extract these tarballs\nback into platform-specific directories](https://github.com/kubernetes/kubernetes/blob/193f659a1cd454b93cbe1e7b1f13b77c21783461/cluster/get-kube-binaries.sh#L143-L156)\nto support downloading multiple platforms into a single workspace.\n\nThe test tarball should follow the lead of the other platform-specific tarballs\nand place the binaries under `test/bin`. We can then reuse the existing\nfunctionality already implemented for the other tarballs.\n\n#### Portable sources\n\nPortable sources are basically copied directly from the source tree:\n\n- `test/e2e/testing-manifests/`\n- `test/images/`\n- `test/kubemark/`\n- `hack/` ([partially](https://github.com/kubernetes/kubernetes/blob/193f659a1cd454b93cbe1e7b1f13b77c21783461/hack/lib/golang.sh#L193-L197))\n\nWe have two options for these:\n\n1. Continue to distribute as a separate tarball, either `kubernetes-test.tar.gz`,\n   or possibly something like `kubernetes-test-portable.tar.gz`.\n- This makes the distinction very clear vs. the binary artifacts\n- There's already some precedent, such as the `kubernetes-manifest.tar.gz` tarball\n- It slightly complicates downloading of test dependencies\n2. Duplicate these sources into each binary-specific tarball.\n- Simplifies test dependency distribution - may only need to download one\n  tarball if client and server are same platform\n- Portable sources are small (as of v1.13.2, approximately 2.7MB uncompressed\n  or about 186KB compressed) so duplication isn't a huge worry\n- Complicates extraction of tarballs with existing scripts, since they assume\n  everything is platform-specific\n\nWe propose the first option as slightly preferable given the tradeoffs.\nSince we intend to continue distributing the mondo test tarball over a\ndeprecation period, we'll use the name `kubernetes-test-portable.tar.gz` for the\nportable sources.\n\n### Updating dependencies on `kubernetes-test.tar.gz`\n\nCurrently the CI workflows and `kubetest` use the `cluster/get-kube.sh` and\n`cluster/get-kube-binaries.sh` scripts to download all artifacts, and\nconveniently `get-kube-binaries.sh` is versioned with the release artifacts in\n`kubernetes.tar.gz`, so simply making `get-kube-binaries.sh` aware of the new\ntarballs should be sufficient for most CI and developer needs.\n\nBecause the test tarball includes binaries used both on the host running tests\n(such as the `e2e` binary), as well as binaries which may run the nodes\n(`e2e.node`), we would need to make sure to download binary test artifacts\ntargeting the host platform, node platform, and possibly server platform.\n\nA quick search reveals a few other uses of `kubernetes-test.tar.gz`, mostly in\n`cluster/`. We can update these to use the platform-specific tarballs, possibly\nwith a fallback to the mondo-tarball if worried about versioning.\n\n#### Dependencies outside the Kubernetes organization\n\nSearching GitHub for references to `kubernetes-test.tar.gz` largely returns\nforks of the main kubernetes repository (including some very old forks,\nidentifiable by the script `e2e-from-release.sh`). Since these forks are not\nlikely to depend on upstream release artifacts, we can ignore them.\n\nThe Samsung SDS CNCT kraken-lib repository has a reference to `kubernetes-test.tar.gz`\nin its [conformance test script](https://github.com/samsung-cnct/kraken-lib/blob/aceab16c316bafcdb1f542dc67876dd2e5279f6b/build-scripts/conformance-tests#L16),\nbut this repo is also marked deprecated and read-only, and there have been no\nchanges since July 2018.\n\nIn vmware/simple-k8s-test-env, the `sk8.sh` file uses\n[`kubernetes-test.tar.gz`](https://github.com/vmware/simple-k8s-test-env/blob/master/sk8.sh#L4481),\nand this repo seems actively maintained, so we should make sure this continues\nworking.\n\nThe [reference](https://github.com/knative/test-infra/blob/8ef3dc1c2ed07e64024bc68c9dbd1a2e10e9e975/scripts/e2e-tests.sh#L118-L120)\nto `kubernetes-test.tar.gz` in knative/test-infra is hilarious.\n\nThere may be other uses that are not easily identifiable, so we will follow a\ndeprecation process of the mondo test tarball as described in the next section.\n\n### Risks and Mitigations\n\nIt's hard to tell who uses these test tarballs outside the core project or\nwithout tools like `kubetest`. We'll need to broadcast this change widely\nso that any downstream users are aware of the incompatible changes.\n\nAs this is an inherently breaking change, we must decide when to\ncause the break. Assuming this effort is targeted for the 1.14 release:\n\n1. We can continue to produce a mondo-tarball for 3 releases, along with new\n   split tarballs; i.e., both 1.14 and 1.15 would contain both split and mondo\n   test tarballs, while 1.16 would only use a split test tarball. This way one\n   could continue to use the mondo tarball through the 1.15 release cycle,\n   and then switch to using split test tarballs for 1.16, as all supported\n   releases would then be producing split test tarballs.\n2. We can make a clean break for 1.14, not producing any mondo test tarballs.\n   Downstream users would need to account for the break immediately,\n   and would also need to special-case for older releases that still use the\n   mondo test tarball.\n3. A somewhat hybrid approach, mixing 1 and 2 backwards in time:\n   a. Produce both mondo test tarballs and split test tarballs on master for\n      a few weeks.\n   b. Backport split tarballs to older releases still supported (1.11 through\n      1.13), but continue to produce mondo test tarballs. We would never remove\n      the mondo test tarballs from these releases, instead continuing to\n      produce both.\n   c. Update all test infrastructure to use split test tarballs\n   d. Remove the mondo test tarball from 1.14 before the first beta release.\n\nGiven the Kubernetes [deprecation policy](https://kubernetes.io/docs/reference/using-api/deprecation-policy/),\nwe should go for option 1 and continue to distribute mondo and split\ntest tarballs for the 1.14 release, and possibly for several releases\nthereafter. (It's not entirely clear exactly which deprecation policy applies\nto this change, however.)\n\nWe'll mark the mondo test tarball as deprecated in the 1.14 release, both\nthrough announcements in the release notes, as well as a `DEPRECATION` notice\nin the mondo test tarball.\n\n### Test Plan\n\nWe'll start by building both the mondo test tarball and split test tarballs in\nCI, followed by updating test infrastructure to use the new split tarballs.\nWe will monitor TestGrid jobs to ensure that nothing is noticeably broken by\nthe change, and our primary sources will be those on the\n[sig-release-master-blocking](https://testgrid.k8s.io/sig-release-master-blocking),\n[sig-release-master-informing](https://testgrid.k8s.io/sig-release-master-informing),\nand [sig-release-master-upgrade](https://testgrid.k8s.io/sig-release-master-upgrade)\ndashboards.\n\nWe'll also reach out to community members testing on non-amd64 architectures,\nsince they're most likely to be impacted by this change.\n\nWe'll work with any downstream consumers we can find to update them to use the\nsplit tarballs ahead of the 1.14.0 release, but will continue to support\nthe mondo test tarball through at least 1.14's complete lifecycle.\n\n### Graduation Criteria\n\nTo consider this effort complete, we should no longer be distributing a\nmondo-tarball of test artifacts, and all TestGrid dashboards should show a\nsimilar level of greenness.\n\nWhile ideally we'd make a clean break, removing the mondo-tarball at the same\ntime as we create the platform-specific test tarballs, to ensure a smoother\nrollout we will distribute both the split and mondo test tarballs for a while,\nand this effort will not be deemed complete until the mondo test tarball is\ngone.\n\n## References\n\nSimilar discussion and work on the other release tarballs:\n\n- [Release tarballs are too big](https://github.com/kubernetes/kubernetes/issues/28435)\n- [Build release tars per-architecture](https://github.com/kubernetes/kubernetes/issues/28629)\n- [Stop including arch-specific binaries in kubernetes.tar.gz](https://github.com/kubernetes/kubernetes/pull/35737)\n- [Implicitly call cluster/get-kube-binaries.sh](https://github.com/kubernetes/kubernetes/issues/38725)\n- [kubernetes-dev announcement about removing arch-specific binaries from kubernetes.tar.gz \"full\" tarball](https://groups.google.com/d/msg/kubernetes-dev/n9H9I8TrOT4/1cyV5r9fAAAJ)\n- [Recording](https://www.youtube.com/watch?v=WbqRursx39k\u0026t=13m28s) and\n  [notes](https://docs.google.com/document/d/1z8MQpr_jTwhmjLMUaqQyBk1EYG_Y_3D4y4YdMJ7V1Kk/edit#heading=h.1fpwoneimh52)\n  from sig-testing weekly meeting\n\n## Implementation History\n\n- 2019-01-18: proposal on Slack and creation of the KEP\n- 2019-01-28: KEP announced on sig-testing and sig-release mailing lists\n- 2019-01-29: discussion at sig-testing weekly meeting\n- 2019-02-14: implementation https://github.com/kubernetes/kubernetes/pull/74065\n  created, deprecation notice included in mondo test tarball\n- 2019-02-22: implementation https://github.com/kubernetes/kubernetes/pull/74065\n  merged\n"
+  },
+  {
+    "id": "09809effc0c2db0f398cad631df452c8",
+    "title": "Presubmit config inside the tested repo",
+    "authors": ["@alvaroaleman"],
+    "owningSig": "sig-testing",
+    "participatingSigs": ["sig-testing"],
+    "reviewers": ["@stevekuznetsov", "@cjwagner"],
+    "approvers": ["@stevekuznetsov", "@cjwagner"],
+    "editor": "TBD",
+    "creationDate": "2019-06-04",
+    "lastUpdated": "2019-07-24",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Presubmit config inside the tested repo\n\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Risks and Mitigations](#risks-and-mitigations)\n    - [Security](#security)\n    - [Components that need the \u003ccode\u003ePresubmit\u003c/code\u003e configuration but do not have a \u003ccode\u003egit ref\u003c/code\u003e to work on](#components-that-need-the--configuration-but-do-not-have-a--to-work-on)\n- [Implementation History](#implementation-history)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n**ACTION REQUIRED:** In order to merge code into a release, there must be an issue in [kubernetes/enhancements] referencing this KEP and targeting a release milestone **before [Enhancement Freeze](https://github.com/kubernetes/sig-release/tree/master/releases)\nof the targeted release**.\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e., [kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be updated for the enhancement to be released.\n\n- [ ] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nThis document proposes to change the Prow presubmit handling to optionally version some or\nall presubmits in the same repository that also contains the code that is being tested.\n\n## Motivation\n\nCurrently, all jobs for Prow are configured in a central `infra-config` repository that is\nin most cases distinct from the repositories whose code is being tested in presubmits. This\nposes severall challenges:\n\n* It is not possible to see in a Pull Request that introduces a new presubmit if that presubmit will\n  actually pass. The two workarounds for this are to either have both the job author and the reviewer\n  use two CLIs to create a Pod from the job or to make the Job initially optional, verify it with a\n  test Pull Request to the code repository, then make it mandatory. Both of these workarounds are\n  cumbersome.\n* The same issues apply when doing changes to the config of a presubmit\n* When a project maintains multiple branches, e.G. because there are release branches, the\n  maintainers must remember to create a copy of the presubmit in the `infra-config` repository.\n  Otherwise it is easely possible that the presubmit becomes incompatible with one of the branches\n  it is used in, because somone makes a change to its config and forgets to test against all branches.\n  This is an additional step maintainers have to remember when branching off.\n* If the `test-infra` repository is not public, outside collaborators are unable to change job configs. This\n  may happen if an organization that uses Prow has a mixture of public and private repositories and chooses\n  not to bear the maintenance overhead of multiple Prow instances.\n\n\n### Goals\n\n* It is possible to version some or all presubmits of a given repository inside that repository in a\n  `yaml` file\n* This feature is optional and opt-in\n* The triggering of presubmits on pull request creation or updates continues to work and includes the\n  jobs that are managed inside the code repository\n* Re-Running tests via the `/retest` command continues to work and includes the jobs that are\n  managed inside the code repository\n* Explicitly executing optional tests via the `/test \u003c\u003cmyjob\u003e\u003e` command continues to work and includes\n  all jobs that are managed inside the code repository\n* All the existing defaulting and validation for Presubmit jobs is being used to default and validate\n  jobs that are managed inside the code repository\n* Pull Requests on which an error occurred during parsing, defaulting or validation of presubmits that\n  are managed in the code repository are not considered as merge candidates by Tide\n* Tide executes the presubmits that are defined inside the code repository when it re-tests\n* Renamed blocking presubmits added via pull request trigger a migration on all in-flight PRs\n* Removed blocking presubmits via pull request trigger a status retire on all in-flight PRs\n* All existing functionality except for what is listed in the [Risks and Mitigations](#Risks-and-Mitigations) section will continue to work when `inrepoconfig` is enabled\n* All existing functionality will continue to work when `inrepoconfig` is not enabled\n\n### Non-Goals\n\n* The option to configure Postsubmits or Periodics inside the tested repository. This may be\n  done in a future iteration.\n\n## Proposal\n\nIt is proposed to introduce a the option to configure additional presubmits inside\nthe code repositories that are tested by prow via a file named `prow.yaml`.\n\nThis requires to change the existing `Config` struct to not expose a `Presubmits`\nproperty anymore, but instead getter functions to get all Presubmits with the\nones from the `prow.yaml` added, if applicable.\n\nAdditionally, all components that need to access the `Presubmit` configuration need\nto be changed to use the new getters and  to contain a git client which can be used\nto fetch the `Presubmit` config from inside the repo.\n\n### Risks and Mitigations\n\n#### Security\n\nThe current attack vector to get credentials out of Prow is a rogue pull requestor\nchanging the scripts that are being executed during a test run to print or upload\ncredentials that are passed into the job.\n\nWith `inrepoconfig` a pull requestor could additionally create new Jobs that use\ncredentials that are previously not passed into any job of the repo or that are\nexecuted on a different Kubernetes cluster which contains higher-privilege credentials.\n\nBoth the exiting attack vector and the changes introduced via `inrepoconfig` require\nthe pull requestor to be an org member or to get an org member to approve the pull\nrequest for testing.\n\nThere are several possible approaches to mitigate the added security risk, for\nexample:\n\n* Extend the configuration for `inrepoconfig` to allow/deny specific values for\n\tvarious job properties. Easy to setup, but requires code for every possible\n\tproperty\n* Maintain an allow/deny list for users that are allowed to change job configs\n* Allow operators to configure a webhook, which will then receive all pull request\n\tevents and their changes to `prow.yaml`. The webhook can then allow or deny that.\n\tThis is the most flexible solution and would even allow to connect the permission\n\tmanagement for `inrepoconfig` to a third-party system like LDAP. It has the drawback\n\tthat its more complicated to set up and introduces a new SPOF into Prow\n\nFinding the best solution to mitigate the security risk added by `inrepoconfig` will\nnot be part of its first iteration, because that problem is considered to be much\neasier to solve than finding an agreeable solution on how to implement `inrepoconfig`\nitself.\n\n#### Components that need the `Presubmit` configuration but do not have a `git ref` to work on\n\nComponents that need the `Presubmit` config but do not have a git reference at hand\ncan not work as before with `inrepoconfig` because the list of Presubmits depends on\nthe `ref`. This limitation will be documented.\n\n## Implementation History\n\n* A basic but functioning [prototype](https://github.com/kubernetes/test-infra/pull/12836)\n  for this feature was created that served as initial basis for this KEP.\n* A non-working [sketch pull request](https://github.com/kubernetes/test-infra/pull/13342) that shows which parts of Prow need to be touched\n\tand how the signatures for the newly-added functions look like was created to\n\tbe the basis for a discussion on how exactly an implementation could look like\n* Current work is being tracked via a [GitHub tracking issue](https://github.com/kubernetes/test-infra/issues/13370)\n"
+  },
+  {
+    "id": "64a59b94cddab796389f7c352a39905f",
+    "title": "Windows Group Managed Service Accounts for Container Identity",
+    "authors": ["@ddebroy", "@jeremywx", "@patricklang"],
+    "owningSig": "sig-windows",
+    "participatingSigs": ["sig-auth", "sig-node", "sig-architecture", "sig-docs"],
+    "reviewers": ["@liggitt", "@mikedanese", "@yujuhong", "@patricklang"],
+    "approvers": ["@liggitt", "@yujuhong", "@patricklang"],
+    "editor": "TBD",
+    "creationDate": "2018-11-29",
+    "lastUpdated": "2019-01-25",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Windows Group Managed Service Accounts for Container Identity\n\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [Background](#background)\n    - [What is Active Directory?](#what-is-active-directory)\n    - [What is a Windows service account?](#what-is-a-windows-service-account)\n    - [How is it applied to containers?](#how-is-it-applied-to-containers)\n  - [User Stories](#user-stories)\n    - [Web Applications with MS SQL Server](#web-applications-with-ms-sql-server)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n    - [GMSA specification for pods and containers](#gmsa-specification-for-pods-and-containers)\n    - [GMSAExpander webhook](#gmsaexpander-webhook)\n    - [GMSAExpander and GMSAAuthorizer Webhooks](#gmsaexpander-and-gmsaauthorizer-webhooks)\n    - [Changes in Kubelet/kuberuntime for Windows:](#changes-in-kubeletkuberuntime-for-windows)\n    - [Changes in CRI API:](#changes-in-cri-api)\n    - [Changes in Dockershim](#changes-in-dockershim)\n    - [Changes in CRIContainerD](#changes-in-cricontainerd)\n    - [Changes in Windows OCI runtime](#changes-in-windows-oci-runtime)\n  - [Risks and Mitigations](#risks-and-mitigations)\n    - [Threat vectors and countermeasures](#threat-vectors-and-countermeasures)\n    - [Transitioning from Alpha annotations to Beta/Stable fields](#transitioning-from-alpha-annotations-to-betastable-fields)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Drawbacks [optional]](#drawbacks-optional)\n- [Alternatives](#alternatives)\n  - [Other authentication methods](#other-authentication-methods)\n  - [Injecting credentials from a volume](#injecting-credentials-from-a-volume)\n  - [Specifying only the name of GMSACredentialSpec objects in pod spec fields/annotations](#specifying-only-the-name-of-gmsacredentialspec-objects-in-pod-spec-fieldsannotations)\n  - [Enforce presence of GMSAAuthorizer and RBAC mode to enable GMSA functionality in Kubelet](#enforce-presence-of-gmsaauthorizer-and-rbac-mode-to-enable-gmsa-functionality-in-kubelet)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nActive Directory is a service that is built-in and commonly used on Windows Server deployments for user and computer identity. Apps are run using Active Directory identities to enable common user to service, and service to service authentication and authorization. This proposal aims to support a specific type of identity, Group Managed Service Accounts (GMSA), for use with Windows Server containers. This will allow an operator to choose a GMSA at deployment time, and run containers using it to connect to existing applications such as a database or API server without changing how the authentication and authorization are performed.\n\n## Motivation\n\nThere has been a lot of interest in supporting GMSA for Windows containers since it's the only way for a Windows application to use an Active Directory identity. This is shown in asks and questions from both public \u0026 private conversations:\n\n- https://github.com/kubernetes/kubernetes/issues/51691 \"For windows based containers, there is a need to access shared objects using domain user contexts.\"\n- Multiple large customers are asking the Microsoft Windows team to enable this feature through container orchestrators\n- Multiple developers have blogged how to use this feature, but all are on standalone machines instead of orchestrated through Kubernetes\n  - https://artisticcheese.wordpress.com/2017/09/09/enabling-integrated-windows-authentication-in-windows-docker-container/\n  - https://community.cireson.com/discussion/3853/example-cireson-scsm-portal-on-docker-windows-containers\n  - https://cloudiqtech.com/windows-2016-docker-containers-using-gmsa-connect-to-sql-server/\n  - https://success.docker.com/api/asset/.%2Fmodernizing-traditional-dot-net-applications%2F%23integratedwindowsauthentication \n\n\n### Goals\n\n- Windows users can run containers with an existing GMSA identity on Kubernetes\n- No extra files or Windows registry keys are needed on each Windows node. All needed data should flow through Kubernetes+Kubelet APIs\n- Prevent pods from being inadvertently scheduled with service accounts that do not have access to a GMSA\n\n### Non-Goals\n\n- Running Linux applications using GMSA or a general Kerberos based authentication system\n- Replacing any existing Kubernetes authorization or authentication controls. Specifically, a subset of users cannot be restricted from creating pods with Service Accounts authorized to use certain GMSAs within a namespace if the users are already authorized to create pods within that namespace. Namespaces serve as the ACL boundary today in Kubernetes and we do not try to modify or enhance this in the context of GMSAs to prevent escalation of privilege through a service account authorized to use certain GMSAs.\n- Providing unique container identities. By design, Windows GMSA are used where multiple nodes are running apps as the same Active Directory principal.\n- Isolation between container users and processes running as the GMSA. Windows already allows users and system services with sufficient privilege to create processes using a GMSA.\n- Preventing the node from having access to the GMSA. Since the node already has authorization to access the GMSA, it can start processes or services using as the GMSA. Containers do not change this behavior.\n- Restricting specification of GMSA credential specs in pods or containers if RBAC is not enabled or the admission webhook described below is not installed/enabled.\n\n\n## Proposal\n\n### Background\n\n#### What is Active Directory?\nWindows applications and services typically use Active Directory identities to facilitate authentication and authorization between resources. In a traditional virtual machine scenario, the computer is joined to an Active Directory domain which enables it to use Kerberos, NTLM, and LDAP to identify and query for information about other resources in the domain. When a computer is joined to Active Directory, it is given an unique identity represented as a computer object in LDAP.\n\n#### What is a Windows service account?\nA Group Managed Service Account (GMSA) is a shared Active Directory identity that enables common scenarios such as authenticating and authorizing incoming requests and accessing downstream resources such as a database server, file share, or other workload. It can be used by multiple authorized users or computers at the same time.\n\n#### How is it applied to containers?\nTo achieve the scale and speed required for containers, Windows uses a group managed service account in lieu of individual computer accounts to enable Windows containers to communicate with Active Directory. As of right now, the Host Computer Service (which exposes the interface to manage containers) in Windows cannot use individual Active Directory computer \u0026 user accounts - it only supports GMSA.\n\nDifferent containers on the same machine can use different GMSAs to represent the specific workload they are hosting, allowing operators to granularly control which resources a container has access to. However, to run a container with a GMSA identity, an additional parameter must be supplied to the Windows Host Compute Service to indicate the intended identity. This proposal seeks to add support in Kubernetes for this parameter to enable Windows containers to communicate with other enterprise resources.\n\nIt's also worth noting that Docker implements this in a different way that's not managed centrally. It requires dropping a file on the node and referencing it by name, eg: docker run --credential-spec='file://gmsa-cred-spec1.json' . For more details see the Microsoft doc.\n\n\n### User Stories\n\n\n#### Web Applications with MS SQL Server\nA developer has a Windows web app that they would like to deploy in a container with Kubernetes. For example, it may have a web tier that they want to scale out running ASP.Net hosted in the IIS web server. Backend data is stored in a Microsoft SQL database, which all of the web servers need to access behind a load balancer. An Active Directory Group Managed Service Account is used to avoid hardcoded passwords, and the web servers run with that credential today. The SQL Database has a user with read/write access to that GMSA so the web servers can access it. When they move the web tier into a container, they want to preserve the existing authentication and authorization models.\n\nWhen this app moves to production on containers, the team will need to coordinate to make sure the right GMSA is carried over to the container deployment.\n\n1. An Active Directory domain administrator will:\n\n  - Join all Windows Kubernetes nodes to the Active Directory domain.\n  - Provision the GMSA and gives details to Kubernetes cluster admin.\n  - Assign access to a AD security group to control what machines can use this GMSA. The AD security group should include all authorized Kubernetes nodes.\n\n2. A Kubernetes cluster admin (or cluster setup tools or an operator) will install a cluster scoped CRD (of kind GMSACredentialSpec) whose instances will be used to store GMSA credential spec configuration:\n\n```\napiVersion: apiextensions.k8s.io/v1beta1\nkind: CustomResourceDefinition\nmetadata:\n  name: gmsacredentialspecs.windows.k8s.io\nspec:\n  group: windows.k8s.io\n  version: v1alpha1\n  names:\n    kind: GMSACredentialSpec\n    plural: gmsacredentialspecs\n  scope: Cluster\n  validation:\n    openAPIV3Schema:\n      properties:\n        credspec:\n          description: GMSA Credential Spec\n          type: object\n```\n\nA GMSACredentialSpec may be used by sidecar containers across different namespaces. Therefore the CRD needs to be cluster scoped.\n\n3. A Kubernetes cluster admin will create a GMSACredentialSpec object populated with the credential spec associated with a desired GMSA:\n\n  - The cluster admin may run a utility Windows PowerShell script to generate the YAML definition of a GMSACredentialSpec object populated with the GMSA credential spec details. Note that the credential spec YAML follows the structure of the credential spec (in JSON) as referred to in the [OCI spec](https://github.com/opencontainers/runtime-spec/blob/master/config-windows.md#credential-spec). The utility Powershell script for generating the YAML will be largely identical to the already published [Powershell script](https://github.com/MicrosoftDocs/Virtualization-Documentation/blob/live/windows-server-container-tools/ServiceAccounts/CredentialSpec.psm1) with the following differences: [a] it will output the credential spec in YAML format and [b] it will encapsulate the credential spec data within a Kubernetes object YAML (of kind GMSACredentialSpec). The GMSACredentialSpec YAML will not contain any passwords or crypto secrets. Example credential spec YAML for a GMSA webapplication1:\n\n```\napiVersion: windows.k8s.io/v1alpha1\nkind: GMSACredentialSpec\nmetadata:\n  name: \"webapp1-credspec\"\ncredspec:\n  ActiveDirectoryConfig:\n    GroupManagedServiceAccounts:\n    - Name: WebApplication1\n      Scope: CONTOSO\n    - Name: WebApplication1\n      Scope: contoso.com\n  CmsPlugins:\n  - ActiveDirectory\n  DomainJoinConfig:\n    DnsName: contoso.com\n    DnsTreeName: contoso.com\n    Guid: 244818ae-87ca-4fcd-92ec-e79e5252348a\n    MachineAccountName: WebApplication1\n    NetBiosName: CONTOSO\n    Sid: S-1-5-21-2126729477-2524075714-3094792973\n```\n  \n  - With the YAML from above (or generated manually), the cluster admin will create a GMSACredentialSpec object in the cluster.\n\n4. A Kubernetes cluster admin will configure RBAC for the GMSACredentialSpec so that only desired service accounts can use the GMSACredentialSpec:\n\n  - Create a cluster wide role to get and \"use\" the GMSACredentialSpec:\n\n```\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n  name: webapp1-gmsa-user\nrules:\n- apiGroups: [\"windows.k8s.io\"]\n  resources: [\"gmsacredentialspecs\"]\n  resourceNames: [\"webapp1-credspec\"]\n  verbs: [\"get\", \"use\"]\n```\n\n  - Create a rolebinding and assign desired service accounts to the above role:\n\n```\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n  name: use-webapp1-gmsa\n  namespace: default\nsubjects:\n- kind: ServiceAccount\n  name: webapp-sa\n  apiGroup: rbac.authorization.k8s.io\nroleRef:\n  kind: ClusterRole\n  name: webapp1-gmsa-user\n  apiGroup: rbac.authorization.k8s.io\n```\n  \n5. Application admins will deploy app pods that require a GMSA identity along with a Service Account authorized to use the GMSAs. There will be two ways to specify the GMSA credential spec details for pods and containers. It is expected that users will typically use the first option as it is more user friendly. The second option is available mainly due to an artifact of the design choices made and described here for completeness.\n\n  - Specify the name of the desired GMSACredentialSpec object (e.g. `webapp1-credspec`): If an application administrator wants containers to be initialized with a GMSA identity, specifying the names of the desired GMSACredentialSpec objects is mandatory. In the alpha stage of this feature, the name of the desired GMSACredentialSpec can be set through an annotation on the pod (applicable to all containers): `pod.alpha.windows.kubernetes.io/gmsa-credential-spec-name` as well as for each container through annotations of the form: `\u003ccontainerName\u003e.container.alpha.windows.kubernetes.io/gmsa-credential-spec-name`. In the beta stage, the annotations will be superseded by fields in the securityContext of the pod: `podspec.securityContext.windows.gmsaCredentialSpecName` and in the securityContext of each container:  `podspec.container[i].securityContext.windows.gmsaCredentialSpecName`. The GMSACredentialSpec name for a container will override the GMSACredentialSpec name specified for the whole pod. Sample pod spec showing specification of GMSACredentialSpec name at the pod level and overriding it for one of the containers:\n\n```\napiVersion: v1\nkind: Pod\nmetadata:\n  name: iis\n  labels:\n    name: iis\n  annotations: {\n    pod.alpha.windows.kubernetes.io/gmsa-credential-spec-name : webapp1-credspec\n    iis.container.alpha.windows.kubernetes.io/gmsa-credential-spec-name : webapp2-credspec\n  }\nspec:\n  containers:\n    - name: iis\n      image: mcr.microsoft.com/windows/servercore/iis:windowsservercore-ltsc2019\n      ports:\n        - containerPort: 80\n    - name: logger\n      image: eventlogger:2019\n      ports:\n        - containerPort: 80\n  nodeSelector:\n    beta.kubernetes.io/os : windows\n```\n\n  - Specify the contents of the `credspec` field of GMSACredentialSpec that gets passed down to the container runtime: Specifying the credential spec contents in JSON form is optional and unnecessary. GMSAExpander will automatically populate this field (as described in the next section) based on the name of the GMSACredentialSpec object. In the alpha stage of this feature, a JSON representation of the contents of the desired GMSACredentialSpec may be set through an annotation on the pod (applicable to all containers): `pod.alpha.windows.kubernetes.io/gmsa-credential-spec` as well as for each container through annotations of the form: `\u003ccontainerName\u003e.container.alpha.windows.kubernetes.io/gmsa-credential-spec`. In the beta stage and beyond, the annotations will be superseded by a field the securityContext of the pod `podspec.securityContext.windows.gmsaCredentialSpec` and in the securityContext of each  container: `podspec.container[i].securityContext.windows.gmsaCredentialSpec`. The credential spec JSON for a container will override the credential spec JSON specified for the whole pod.\n\n  The ability to specify credential specs for each container within a pod aligns with how security attributes like `runAsGroup`, `runAsUser`, etc. can be specified at the pod level and overridden at the container level if desired.\n  \n  Note that as this feature graduates to Beta, support for the annotations will be removed in favor of securityContext fields in podspec. The implication of the removal of support for the Alpha annotations is covered in the Risks and Mitigations section later.\n\n6. A mutating webhook admission controller, GMSAExpander, will act on pod creations. GMSAExpander will look up the GMSACredentialSpec object referred to by name and use the contents in the `credspec` field to populate the GMSA credential spec JSON if absent or empty in the necessary annotations [in Alpha] or securityContext fields [Beta onwards]. Specifics of the checks performed, fields affected and error scenarios for GMSAExpander is covered in details in the Implementation section below.\n\n7. A validating webhook admission controller, GMSAAuthorizer, will act on pod creations (as well as updates as discussed later in Step 11) and execute a series of checks and authorization around the GMSA annotations [in Alpha] or securityContext fields [in Beta]. Specifics of the annotations or securityContext fields examined and authorizations checks performed along with error scenarios for GMSAAuthorizer is covered in details in the Implementation section below.\n\n8. Kubelet.exe in Windows nodes will examine the credential spec related annotations [in Alpha] or securityContext fields [Beta onwards] for a given pod as well as for each container in the pod. For each container, Kubelet will compute an effective credential spec - either the credential spec specified for the whole pod or a credential spec specified specifically for the container. During Alpha, Kubelet will set the effective credential spec for each container as annotation: `\u003ccontainerName\u003e.container.alpha.windows.kubernetes.io/gmsa-credential-spec`. Beta onwards, Kubelet will set the effective credential spec for each container in a new security context field `WindowsContainerSecurityContext.CredentialSpec` which will require an update to the CRI API. Please see the Implementation section below for details on the enhancements necessary in Kubelet and CRI API.\n\n9. The Windows CRI implementation will access the credential spec JSON through annotations [`\u003ccontainerName\u003e.container.alpha.windows.kubernetes.io/gmsa-credential-spec` for Alpha] or securityContext field [`WindowsContainerSecurityContext.CredentialSpec` Beta onwards] in `CreateContainerRequest` for each container in a pod. The CRI implementation will transmit the credential spec JSON through a runtime implementation dependent mechanism to a specific container runtime. For example:\n - Docker (to be supported in Alpha): will receive the path to a file created on the node's file system under `C:\\ProgramData\\docker\\CredentialSpecs\\` and populated by dockershim with the credential spec JSON. Docker will read the contents of the credential spec file and pass it to Windows Host Compute Service (HCS) when creating and starting a container in Windows.\n - ContainerD (to be supported in Beta): will receive a OCI Spec with [windows.CredentialSpec]( https://github.com/opencontainers/runtime-spec/blob/master/config-windows.md#credential-spec) populated by CRIContainerD with the credential spec JSON. The OCI spec will be passed to a OCI runtime like RunHCS.exe when starting a container.\nPlease see the Implementation section below for details on the enhancements necessary for select CRI implementations and their corresponding runtimes.\n\n10. The Windows container runtime (Docker or ContainerD + RunHCS) will depend on Windows Host Compute Service (HCS) APIs/vmcompute.exe to start the containers and assign them user identities  corresponding to the GMSA configured for each container. Using the GMSA identity, the processes within the container can authenticate to a service protected by GMSA like database. The containers that fail to start due to invalid GMSA configuration (as determined by HCS or container runtime like Docker) will end up in a failed state with the following when described:\n```\nState:              Waiting\n      Reason:           CrashLoopBackOff\nLast State:         Terminated\n      Reason:           ContainerCannotRun\n      Message:          [runtime specific error e.g. \"encountered an error during CreateContainer:...\"]\n```\n\nThere are a couple of failure scenarios to consider in the context of incorrect GMSA configurations for containers and hosts where containers do get started in spite of incorrect GMSA configuration:\n\n - On a Windows node not connected to AD, a GMSA credential spec JSON is passed that is structurally valid JSON: Windows HCS APIs are not able to detect the fact that the Windows node is not connected to a domain and starts the container normally.\n - On a Windows node connected to AD, a GMSA credential spec JSON is passed that the node is not configured to use: Windows HCS APIs are not able to detect the fact that the Windows node is not authorized to use the GMSA and starts the container.\n\nIn the above scenarios, after a container has started with the GMSA credential spec JSON configured, within the container, validation commands like `whoami /upn` fails with `Unable to get User Principal Name (UPN) as the current logged-on user is not a domain user` and `nltest /query` fails with `ERROR_NO_LOGON_SERVERS`/`ERROR_NO_TRUST_LSA_SECRET`. An init container is recommended to be added to pod specs (where GMSA configurations are desired) to validate that the desired domain can be reached (through `nltest /query`) as well as the desired identity can be obtained (through `whoami /upn`) with the configured GMSA credential specs. If the validation commands fail, the init container will exit with failures and thus prevent pod bringup. The failure can be discovered by describing the pod and looking for errors along the lines of the following for the GMSA validating init container:\n```\nState:           Waiting\n      Reason:        CrashLoopBackOff\nLast State:      Terminated\n      Reason:        Error\n```\n\n11. During any pod update, any changes to a pod's `securityContext` will be blocked (as is the case today) by `ValidatePodUpdate` Beta onwards. Updates of the annotations associated with GMSA will be rejected by GMSAAuthorizer in Alpha stage. Note that modifications or removal of the named GMSACredentialSpec, or removal of authorization does not disrupt update/deletion of previously created pods.\n\n\n### Implementation Details/Notes/Constraints [optional]\n\n#### GMSA specification for pods and containers\nIn the Alpha phase of this feature we will use the following annotations on a pod for GMSA credential spec:\n  - References to names of GMSACredentialSpec objects will be specified through the following annotations:\n    - At the pod level: `pod.alpha.windows.kubernetes.io/gmsa-credential-spec-name`\n    - At the container level: `\u003ccontainerName\u003e.container.alpha.windows.kubernetes.io/gmsa-credential-spec-name`\n  - The contents of the credential spec will be specified through or populated in the following annotations:\n    - At the pod level: `pod.alpha.windows.kubernetes.io/gmsa-credential-spec`\n    - At the container level: `\u003ccontainerName\u003e.container.alpha.windows.kubernetes.io/gmsa-credential-spec`\n\nIn the Beta phase of this feature, support for above the annotations above will be dropped. The annotations will be superseded by fields in the pod spec:\n  - References to names of GMSACredentialSpec objects will be specified through the following fields:\n    - At the pod level `podspec.securityContext.windows.gmsaCredentialSpecName`\n    - At the container level: `podspec.container[i].securityContext.windows.gmsaCredentialSpecName`\n  - The contents of the credential spec will be specified through or populated in the following fields:\n    - At the pod level: `podspec.securityContext.windows.gmsaCredentialSpec`\n    - At the container level: `podspec.container[i].securityContext.windows.gmsaCredentialSpec`\n\nNote that the `windows.gmsaCredentialSpecName` and `windows.gmsaCredentialSpec` fields of the `securityContext` struct is speculative at the moment and may change in the future. The names/parents of the GMSA fields will depend on how the exact structure and representation of OS specific `securityContext` fields evolve.\n\n#### GMSAExpander webhook\nA new webhook, GMSAExpander, will be implemented and configured to act on pod creation. It will perform the following steps:\n\n  - In Alpha, check if GMSA credential spec JSON annotations [`pod.alpha.windows.kubernetes.io/gmsa-credential-spec` or `\u003ccontainerName\u003e.container.alpha.windows.kubernetes.io/gmsa-credential-spec`] corresponding to each reference to names of GMSACredentialSpec objects in annotations [`pod.alpha.windows.kubernetes.io/gmsa-credential-spec-name` or `\u003ccontainerName\u003e.container.alpha.windows.kubernetes.io/gmsa-credential-spec-name/container-name`] is present and populated.\n\n  - In Beta, check if securityContext field [`podspec.securityContext.windows.gmsaCredentialSpec` or `podspec.container[i].securityContext.windows.gmsaCredentialSpec`] corresponding to each reference to names of GMSACredentialSpec objects in securityContext fields [`podspec.securityContext.windows.gmsaCredentialSpecName` or `podspec.container[i].securityContext.windows.gmsaCredentialSpecName` is populated.\n\n  - If the GMSA credential spec JSON annotation is absent or empty or the securityContext field is empty, look up the GMSACredentialSpec object by name. If GMSACredentialSpec object does not exist, return error 422 Unprocessable Entity with message indicating GMSACredentialSpec object with specified name could not be found. If the GMSACredentialSpec object exists, obtain the data in the `credspec` member and convert it to JSON. Next, create a new annotation [`pod.alpha.windows.kubernetes.io/gmsa-credential-spec` or `\u003ccontainerName\u003e.container.alpha.windows.kubernetes.io/gmsa-credential-spec`] if absent and populate it with the JSON (for Alpha) or populate the securityContext field [`podspec.securityContext.windows.gmsaCredentialSpec` or `podspec.container[i].securityContext.windows.gmsaCredentialSpec`] with the JSON (Beta onwards).\n\nNote that the annotations will not be processed/populated once the feature graduates to Beta and the securityContext fields will be used instead.\n\n#### GMSAExpander and GMSAAuthorizer Webhooks\nA new webhook, GMSAAuthorizer will be implemented to act on pod creation and updates and perform several checks and validations.\n\nDuring pod creation, the following checks and validations will be executed:\n\n  - Authorize the service account specified for the pod to use specified GMSACredentialSpec objects: First look up all references to GMSACredentialSpec objects by name specified through annotations [`pod.alpha.windows.kubernetes.io/gmsa-credential-spec-name` and `\u003ccontainerName\u003e.container.alpha.windows.kubernetes.io/gmsa-credential-spec-name` in Alpha] or securityContext fields [`podspec.securityContext.windows.gmsaCredentialSpecName` and `podspec.container[i].securityContext.windows.gmsaCredentialSpecName` Beta onwards]. Next, generate custom `AttributesRecord`s with `verb` set to `use`, `name` set to the specified name of a GMSACredentialSpec object and `user` set to the service account of the pod. Finally, the `AttributesRecord`s will be passed to authorizers to check against RBAC configurations. A failure from the authorizer results in a response 403: Forbidden with message indicating the GMSACredentialSpec object to which the access was denied.\n\n  - Check each GMSA credential spec JSON has an associated reference to a GMSACredentialSpec object by name: The GMSA credential spec JSON may be populated in annotations or securityContext fields directly by app admins or through GMSAExpander. For each GMSA credential spec JSON specification in annotations [`pod.alpha.windows.kubernetes.io/gmsa-credential-spec` and `\u003ccontainerName\u003e.container.alpha.windows.kubernetes.io/gmsa-credential-spec` in Alpha] or securityContext fields [`podspec.securityContext.windows.gmsaCredentialSpec` or `podspec.container[i].securityContext.windows.gmsaCredentialSpec` Beta onwards], locate a corresponding reference to a GMSACredentialSpec object by name in annotations [`pod.alpha.windows.kubernetes.io/gmsa-credential-spec-name` or `\u003ccontainerName\u003e.container.alpha.windows.kubernetes.io/gmsa-credential-spec-name` in Alpha] or securityContext fields [`podspec.securityContext.windows.gmsaCredentialSpecName` or `podspec.container[i].securityContext.windows.gmsaCredentialSpecName` Beta onwards]. If the reference to a GMSACredentialSpec object by name is not found, pod creation will be failed with 422: Unprocessable Entity along with a message indicating the GMSA credential spec JSON whose name was absent. If the reference is found, establish pair \u003cGMSA CredentialSpec object name, GMSA credential spec JSON\u003e.\n\n  - Validate contents of GMSA credential spec JSON: For each \u003cGMSA CredentialSpec object name, GMSA credential spec JSON\u003e pair established above, compare, in a deep equal fashion, the contents of the `credspec` member of the GMSACredentialSpec object (referred to by name) with the GMSA credential spec JSON obtained from annotations/securityContext fields. If the deep equal comparison fails, pod creation will be failed with 422: Unprocessable Entity along with a message indicating the mismatch and the contents of the credential spec JSONs that did not match.\n\nDuring pod updates, changes to the credential spec annotations will be blocked by GMSAAuthorizer and failed with 400: BadRequest. Note that modifications or removal of the named GMSACredentialSpec, or removal of authorization does not disrupt update/deletion of previously created pods.\n\nNote that the annotations will not be processed/populated once the feature graduates to Beta and the securityContext fields will be used instead.\n\nIf the GMSAAuthorizer webhook is not installed and configured, no authorization checks will be performed on the contents of the credential spec JSON. This will allow arbitrary credential spec JSON to be specified for pods/containers and sent down to the container runtime. Therefore when configuring Windows worker nodes for GMSA support, in the Alpha stage, Kubernetes cluster administrators need to ensure that the GMSAAuthorizer webhook is installed and configured.\n\n#### Changes in Kubelet/kuberuntime for Windows: \n\nIn the Alpha phase, `applyPlatformSpecificContainerConfig` will be enhanced (under a feature flag: `WindowsGMSA`) to analyze the credential spec related annotations on the pod [`pod.alpha.windows.kubernetes.io/gmsa-credential-spec` and `\u003ccontainerName\u003e.container.alpha.windows.kubernetes.io/gmsa-credential-spec`] and determine an effective credential spec for each container:\n - If `\u003ccontainerName\u003e.container.alpha.windows.kubernetes.io/gmsa-credential-spec` is populated, effective credential spec of the container is set to that value.\n - If `\u003ccontainerName\u003e.container.alpha.windows.kubernetes.io/gmsa-credential-spec` is absent but `pod.alpha.windows.kubernetes.io/gmsa-credential-spec` is populated, effective credential spec of the container is set to the contents of `pod.alpha.windows.kubernetes.io/gmsa-credential-spec`.\n - If `\u003ccontainerName\u003e.container.alpha.windows.kubernetes.io/gmsa-credential-spec` is absent and `pod.alpha.windows.kubernetes.io/gmsa-credential-spec` is absent effective credential spec is nil.\nNext, annotation: `container.alpha.windows.kubernetes.io/gmsa-credential-spec` in `ContainerConfig` will be populated with the effective credential spec of the container.\n\nIn the Beta phase, the logic in `applyPlatformSpecificContainerConfig` to populate annotation `container.alpha.windows.kubernetes.io/gmsa-credential-spec` in `ContainerConfig` will be removed. Instead, `DetermineEffectiveSecurityContext` will be enhanced (also under a feature flag: `WindowsGMSA`) to analyze the `securityContext.windows.gmsaCredentialSpec` fields for the pod overall and each container in the podspec and determine an effective credential spec for each container in the same fashion described above (and as it does today for several fields like RunAsUser, etc). Next, `ContainerConfig.WindowsContainerSecurityContext.CredentialSpec` will be populated with the effective credential spec for the container.\n\n#### Changes in CRI API:\n\nIn the Alpha phase, no changes will be required in the CRI API. Annotation `container.alpha.windows.kubernetes.io/gmsa-credential-spec` in `ContainerConfig` will contain the credential spec JSON associated with each container.\n\nIn the Beta phase, a new field `CredentialSpec String` will be added to `WindowsContainerSecurityContext` in `ContainerConfig`. This field will be populated with the credential spec JSON of a Windows container by Kubelet.\n\n#### Changes in Dockershim\n\nThe GMSA credential spec will be passed to Docker through temporary entries in the Windows registry under SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Virtualization\\Containers\\CredentialSpecs. The registry entries will be created with unique key names that have a common prefix. The contents of the registry entries will be used to populate `HostConfig.SecurityOpt` with a credential spec file specification. The registry entries will be deleted as soon as `CreateContainer` has been invoked on the Docker client. An alternative implementation considered was to utilize files instead of registry entries but the path and drive where the files can be stored is hard to determine as there could be multiple installations of different versions of Docker engine under different directory paths. \n\nDuring Alpha, `dockerService.CreateContainer` function will be enhanced (under a feature flag: `WindowsGMSA`) to create the temporary registry entries and populate them with the contents of `container.alpha.windows.kubernetes.io/gmsa-credential-spec` annotation in `CreateContainerRequest.ContainerConfig`. Beta onwards, `dockerService.CreateContainer` (under a feature flag: `WindowsGMSA`) will use the contents of `WindowsContainerSecurityContext.CredentialSpec` to populate the registry values.\n\n#### Changes in CRIContainerD\n\nDuring Alpha, updating the CRI API and thus enabling interactions with ContainerD as a runtime is not planned. Once the CRI API has been updated to pass the `WindowsContainerSecurityContext.CredentialSpec` during Beta, CRIContainerD should be able to access the credential spec JSON. At that point, CRIContainerD will need to be enhanced to populate the [windows.CredentialSpec]( https://github.com/opencontainers/runtime-spec/blob/master/config-windows.md#credential-spec) field of the OCI runtime spec for Windows containers with the credential spec JSON passed through CRI.\n\n#### Changes in Windows OCI runtime\n\nThe Windows OCI runtime already has support for `windows.CredentialSpec` and is implemented in Moby/Docker as well hcsshim/runhcs.\n\n### Risks and Mitigations\n\n#### Threat vectors and countermeasures\n\n1. Prevent an unauthorized user from referring to an existing GMSA configmap in the pod spec: The GMSAAuthorizer Admission Controller along with RBAC policies with the `use` verb on a GMSA configmap ensures only users allowed by the kubernetes admin can refer to the GMSA configmap in the pod spec.\n2. Prevent an unauthorized user from using an existing Service Account that is authorized to use an existing GMSA configmap: The GMSAAuthorizer Admission Controller checks the `user` as well as the service account associated with the pod have `use` rights on the GMSA configmap.\n3. Prevent an unauthorized user from reading the GMSA credential spec and using it directly through docker on Windows hosts connected to AD that user has access to: RBAC policy on the GMSA configmaps should only allow `get` verb for authorized users.\n\n#### Transitioning from Alpha annotations to Beta/Stable fields\n\nLogic to process annotations used to specify GMSA details in Alpha phase will be removed once the feature graduates to beta. Since the annotations are only used during the Alpha phase, this deprecation and removal is compliant with [Kubernetes guidelines] (https://kubernetes.io/docs/reference/using-api/deprecation-policy/#deprecating-a-flag-or-cli). When upgrading from a version of Kubernetes with GMSA support in Alpha to a version where GMSA support has graduated to Beta, pod yamls with annotations for GMSA will need to be rewritten to specify the GMSA details in securityContext fields. The opposite conversion will need to be authored in pod YAMLs when downgrading from a version with Beta support of GMSA to Alpha support.\n\n\n## Graduation Criteria\n\n- alpha - Initial implementation with webhook and annotations on pods with no API changes in PodSpec or CRI. Kubelet and Dockershim enhancements will be guarded by a feature flag `WindowsGMSA` and disabled by default. Manual e2e tests with domain joined Window nodes with Docker as the container runtime in a cluster needs to pass.\n- beta - Support for the Alpha annotations will be removed and replaced with new fields in PodSpec and CRI API. GMSA Annotations on a set of pods from a cluster upgraded from a Kubernetes version supporting GMSA configuration in alpha will not be supported and cluster operator will need to rewrite the YAMLs to specify the fields in podspec that supersede the annotations. Removal of support for Alpha annotations is allowed by [Kubernetes guidelines] (https://kubernetes.io/docs/reference/using-api/deprecation-policy/#deprecating-a-flag-or-cli). Feature flag `WindowsGMSA` will be enabled by default for Kubelet and dockershim. Basic e2e test infrastructure in place in Azure leveraging the test suites for Windows e2e along with dedicated DC host VMs. Automated testing will target Docker container run time but some manual testing of ContainerD integration also needs to succeed.\n- ga - e2e tests passing consistently and tests targeting ContainerD/RunHCS  passing as well assuming ContainerD/RunHCS for Windows is stable.\n\n\n## Implementation History\n\nMajor milestones in the life cycle of a KEP should be tracked in `Implementation History`.\nMajor milestones might include\n\n- the `Summary` and `Motivation` sections being merged signaling SIG acceptance\n- the `Proposal` section being merged signaling agreement on a proposed design\n- the date implementation started\n- the first Kubernetes release where an initial version of the KEP was available\n- the version of Kubernetes where the KEP graduated to general availability\n- when the KEP was retired or superseded\n\n## Drawbacks [optional]\n\nWhy should this KEP _not_ be implemented.\n\n## Alternatives \n\n### Other authentication methods\n\nThere are other ways to handle user-service and service-service authentication, but they generally require code changes. This proposal is focused on enabling customers to use existing on-premises Active Directory identity in containers.\n\nFor cloud-native applications, there are other alternatives:\n\n- Kubernetes secrets - if both services are run in Kubernetes, this can be used for username/password or preshared secrets available to each app\n- PKI - If you have a PKI infrastructure, you could choose to deploy application-specific certificates and change applications to trust specific public keys or intermediate certificates\n- Cloud-provider service accounts - there may be other token-based providers available in your cloud. Apps can be modified to use these tokens and APIs for authentication and authorization requests.\n\n### Injecting credentials from a volume\n\nFor certain authentication use cases, a preferred approach may be to surface a volume to the pod with the necessary data that a pod needs to assume an identity injected in the volume. In these cases, the container needs to implement logic to consume and act on the injected data.\n\nIn case of GMSA support, nothing inside the containers of a pod perform any special steps around assuming an identity as that is taken care of by the container runtime at container startup. A container runtime driven solution like GMSA however does require CRI enhancements as mentioned earlier.\n\n### Specifying only the name of GMSACredentialSpec objects in pod spec fields/annotations\n\nTo keep the pod spec changes minimal, we considered having a single field/annotation that specifies the name of the GMSACredentialSpec object (rather than an additional field that is populated with the contents of the credential spec). This approach had the following drawbacks compared to retrieving and storing the credential spec data inside annotations/fields:\n\n- Complicates the Windows CRI code with logic to look up GMSACredentialSpec objects which may be failure prone.\n- Requires the kubelet to be able to access GMSACredentialSpec objects which may require extra RBAC configuration in a locked down environment.\n- Contents of `credspec` in a GMSACredentialSpec object being referred to may change after pod creation. This leads to confusing behavior.\n\n### Enforce presence of GMSAAuthorizer and RBAC mode to enable GMSA functionality in Kubelet\n\nIn order to enforce authorization of service accounts in a cluster before they can be used in conjunction with an approved GMSA, we considered adding checks in the Kubelet layer processing GMSA credential specs. Such enforcement however does not align well with parallel mechanisms and also leads to core Kubernetes code being opinionated about something that may not be necessary.\n\nToday, if PSP or RBAC mode is not configured in a cluster, nothing stops pods with special capabilities from being scheduled. To align with this, we should allow GMSA configurations on pods to be enabled without requiring GMSAAuthorizer to be running and RBAC mode to be enabled.\n\nFurther, decoupling basic GMSA functionality in the Kubelet and CRI layers from authorization keeps the core Kuberenetes code non-opinionated around enforcement of authorization of service accounts for GMSA usage. Kubernetes cluster setup tools as well as Kubernetes distribution vendors can ensure that RBAC mode is enabled and GMSAAuthorizer is configured and installed when Windows nodes joined to a domain are deployed in a cluster.\n\n\n\u003c!-- end matter --\u003e \n\u003c!-- references --\u003e\n[oci-runtime](https://github.com/opencontainers/runtime-spec/blob/master/config-windows.md#credential-spec)\n[manage-serviceaccounts](https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/manage-serviceaccounts)\n"
+  },
+  {
+    "id": "ed1d78b449530489a8c02da9eb322acf",
+    "title": "Windows node support",
+    "authors": ["@astrieanna", "@benmoss", "@patricklang", "@michmike", "@daschott"],
+    "owningSig": "sig-windows",
+    "participatingSigs": ["sig-architecture", "sig-node"],
+    "reviewers": ["sig-architecture", "sig-node", "sig-testing", "sig-release"],
+    "approvers": ["@bgrant0607", "@michmike", "@patricklang", "@spiffxp"],
+    "editor": "TBD",
+    "creationDate": "2018-11-29",
+    "lastUpdated": "2019-03-06",
+    "status": "implemented",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Windows node support\n\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [What works today](#what-works-today)\n  - [Windows Node Roadmap (post-GA work)](#windows-node-roadmap-post-ga-work)\n    - [Custom DNS updates for CNI plugins](#custom-dns-updates-for-cni-plugins)\n  - [What will never work](#what-will-never-work)\n  - [Windows Container Compatibility](#windows-container-compatibility)\n  - [Relevant resources/conversations](#relevant-resourcesconversations)\n  - [Risks and Mitigations](#risks-and-mitigations)\n    - [Ensuring OS-specific workloads land on appropriate container host](#ensuring-os-specific-workloads-land-on-appropriate-container-host)\n    - [Memory Overprovisioning](#memory-overprovisioning)\n- [Graduation Criteria](#graduation-criteria)\n- [Implementation History](#implementation-history)\n- [Testing Plan](#testing-plan)\n  - [Test Dashboard](#test-dashboard)\n  - [Test Environment](#test-environment)\n  - [Test Approach](#test-approach)\n    - [Adapting existing tests](#adapting-existing-tests)\n    - [Substitute test cases](#substitute-test-cases)\n    - [Windows specific tests](#windows-specific-tests)\n- [Conformance Testing](#conformance-testing)\n- [API Reference](#api-reference)\n  - [V1.Container](#v1container)\n  - [V1.Pod](#v1pod)\n  - [V1.PodSecurityContext](#v1podsecuritycontext)\n- [Other references](#other-references)\n\u003c!-- /toc --\u003e\n\n## Summary\n\nThere is strong interest in the community for adding support for workloads running on Microsoft Windows. This is non-trivial due to the significant differences in the implementation of Windows from the Linux-based OSes that have so far been supported by Kubernetes. This KEP will allow Windows nodes to be added to a Kubernetes cluster as compute nodes. With the introduction of Windows nodes, developers will be able to schedule Windows Server containers and run Windows-based applications on Kubernetes.\n\n\n## Motivation\n\nWindows-based workloads still account for a significant portion of the enterprise software space. While containerization technologies emerged first in the UNIX ecosystem, Microsoft has made investments in recent years to enable support for containers in its Windows OS. As users of Windows increasingly turn to containers as the preferred abstraction for running software and modernizing existing applications, the Kubernetes ecosystem stands to benefit by becoming a cross-platform cluster manager.\n\n### Goals\n\n- Enable users to schedule Windows Server containers in Kubernetes through the introduction of support for Windows compute nodes\n- Document the differences and limitations compared to Linux\n- Create a test suite in testgrid to maintain high quality for this feature and prevent regression of functionality \n\n### Non-Goals\n\n- Adding Windows support to all projects in the Kubernetes ecosystem (Cluster Lifecycle, etc)\n- Enable the Kubernetes master components to run on Windows\n- Support for LCOW (Linux Containers on Windows with Hyper-V Isolation)\n\n## Proposal\n\nAs of 29-11-2018 much of the work for enabling Windows nodes has already been completed. Both `kubelet` and `kube-proxy` have been adapted to work on Windows Server, and so the first goal of this KEP is largely already complete. \n\n### What works today\n- Windows-based containers can be created by kubelet, [provided the host OS version matches the container base image](https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/version-compatibility). Microsoft will distribute the operating system-dependent `pause` image (mcr.microsoft.com/k8s/core/pause:1.0.0).\n    - Pod\n      - Single or multiple containers per Pod with process isolation\n      - There are no notable differences in Pod status fields between Linux and Windows containers\n      - Readiness and Liveness probes\n      - postStart \u0026 preStop container lifecycle events\n      - ConfigMap, Secrets: as environment variables or volumes (Volume subpath does not work)\n      - EmptyDir\n      - Named pipe host mounts\n      - Volumes can be shared between containers in a Pod\n      - Resource limits\n    - Services types NodePort, ClusterIP, LoadBalancer, and ExternalName. Service environment variables and headless services work.\n      - Cross operating system service connectivity\n    - Workload controllers ReplicaSet, ReplicationController, Deployments, StatefulSets, DaemonSet, Job, CronJob\n    - Scheduler preemption\n    - Pod \u0026 container metrics\n    - Horizontal Pod Autoscaling using all metrics\n    - KubeCtl Exec\n    - Resource Quotas \n- Windows Server 2019 is the only Windows operating system we will support at GA timeframe. Note above that the host operating system version and the container base image need to match. This is a Windows limitation we cannot overcome.\n- Customers can deploy a heterogeneous cluster, with Windows and Linux compute nodes side-by-side and schedule Docker containers on both operating systems. Of course, Windows Server containers have to be scheduled on Windows and Linux containers on Linux\n- Out-of-tree Pod networking with [Azure-CNI](https://github.com/Azure/azure-container-networking/blob/master/docs/cni.md), [OVN-Kubernetes](https://github.com/openvswitch/ovn-kubernetes), [two CNI meta-plugins](https://github.com/containernetworking/plugins), [Flannel (VXLAN and Host-Gateway)](https://github.com/coreos/flannel) \n- Dockershim CRI\n- Many\u003csup id=\"a1\"\u003e[1]\u003c/sup\u003e of the e2e conformance tests when run with [alternate Windows-based images](https://hub.docker.com/r/e2eteam/) which are being moved to [kubernetes-sigs/windows-testing](https://www.github.com/kubernetes-sigs/windows-testing)\n- Persistent storage: FlexVolume with [SMB + iSCSI](https://github.com/Microsoft/K8s-Storage-Plugins/tree/master/flexvolume/windows), and in-tree AzureFile and AzureDisk providers\n- Kube-Proxy support for L2Bridge and Overlay networks\n\n### Windows Node Roadmap (post-GA work)\n- Group Managed Service Accounts, a way to assign an Active Directory identity to a Windows container, is forthcoming with KEP `Windows Group Managed Service Accounts for Container Identity`. This work will be released as alpha in v1.14 and is already merged.\n- `kubectl port-forward` hasn't been implemented due to lack of an `nsenter` equivalent to run a process inside a network namespace.\n- CRIs other than Dockershim: CRI-containerd support is forthcoming\n- Some kubeadm work was done in the past to add Windows nodes to Kubernetes, but that effort has been dormant since. We will need to revisit that work and complete it in the future.\n- Calico CNI for Pod networking\n- Hyper-V isolation (Currently this is limited to 1 container per Pod and is an alpha feature)\n  - This could enable backwards compatibility likely as a RuntimeClass. This would allow running a container host OS that is newer in version than the container OS (visit this link for additional compatibility definitions https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/version-compatibility)\n- It is unclear if the RuntimeClass proposal from sig-node will simplify scheduled Windows containers. We will work with sig-node on this.\n- Properly implement terminationGracePeriodSeconds for Windows (https://github.com/moby/moby/issues/25982 and https://github.com/kubernetes/kubernetes/issues/73434)\n- Single file mapping and Termination message will work when we introduce CRI containerD support in Windows\n- Design and implement `--enforce-node-allocatable`, hard/soft eviction and `MemoryPressure` conditions. These all depend on cgroups in Linux, and the kubelet will need new work specific to Windows to raise and respond to memory pressure conditions. See [Memory Overprovisioning](#memory-overprovisioning) later in this doc.\n- Fix run_as_username for Windows (https://github.com/kubernetes/kubernetes/issues/73387)\n- Support for Local Traffic Policy and DSR mode on Windows (https://github.com/kubernetes/kubernetes/issues/62046)\n\n\n\n#### Custom DNS updates for CNI plugins\n\nFor v1.14, custom pod DNS configuration tests are not running. Some CNI implementations updates are needed to Azure-CNI, win-bridge, OVN, and flannel which are out of the kubernetes/kubernetes tree. Once those are updated, the tests are tracked in [issue 73414](https://github.com/kubernetes/kubernetes/issues/73414)/[pr 74925](https://github.com/kubernetes/kubernetes/pull/74925) will be merged.\n\nAs part of Azure-CNI [PR#305](https://github.com/Azure/azure-container-networking/pull/305), manual tests were run with Pod.Spec.DNSPolicy = DNSNone. Hostname, Subdomain, and DNSConfig.Nameservers, and DNSConfig.Searches were set correctly based on the Pod spec.\n\nTracking Issues:\n\n- win-bridge [#271](https://github.com/containernetworking/plugins/pull/271) - this is also used in the test passes for GCE, see [gce-k8s-windows-testing#7](https://github.com/yujuhong/gce-k8s-windows-testing/pull/7)\n- Azure-CNI [PR#305](https://github.com/Azure/azure-container-networking/pull/305)\n\n### What will never work\nNote that some features are plain unsupported while some will not work without underlying OS changes\n- Certain Pod functionality\n    - Privileged containers\n    - Pod security context privilege and access control settings. Any Linux Capabilities, SELinux, AppArmor, Seccomp, Capabilities (POSIX Capabilities), and others are not supported \n    - Reservations are not enforced by the OS, but overprovisioning could be blocked with `--enforce-node-allocatable=pods` (pending: tests needed)\n    - Certain volume mappings\n      - Subpath volume mounting\n      - Subpath volume mounting for Secrets\n      - Host mount projection\n      - DefaultMode (due to UID/GID dependency)\n      - readOnly root filesystem. Mapped volumes still support readOnly\n      - Block device mapping\n    - Expanding the mounted volume (resizefs)\n    - HugePages\n    - Memory as the storage medium\n- CSI plugins, which require privileged containers\n- File system features like uui/guid, per-user Linux filesystem permissions, and read-only root filesystems (see note above and also later in the doc about read-only volumes)\n- NFS based storage/volume support (https://github.com/kubernetes/kubernetes/issues/56188)\n- Host networking is not available in Windows\n- ClusterFirstWithHostNet is not supported for DNS. Windows treats all names with a `.` as a FQDN and skips PQDN resolution\n- Not all features of shared namespaces are supported. This is clarified in the API section below\n- The existing node problem detector is Linux-only and requires privileged containers. In general, we don't expect these to be used on Windows because there's no privileged support\n- Overlay networking support in Windows Server 1803 is not fully functional using the `win-overlay` CNI plugin. Specifically service IPs do not work on Windows nodes. This is currently specific to `win-overlay`; other CNI plugins (OVS, AzureCNI) work. Since Windows Server 1803 is not supported for GA, this is mostly not applicable. We left it here since it impacts beta\n- Outbound communication using the ICMP protocol via the `win-overlay`, `win-bridge`, and `Azure-CNI` plugin. Specifically, the Windows data plane ([VFP](https://www.microsoft.com/en-us/research/project/azure-virtual-filtering-platform/)) doesn't support ICMP packet transpositions. This means:\n    - ICMP packets directed to destinations within the same network (e.g. pod to pod communication via ping) will work as expected and without any limitations\n    - TCP/UDP packets will work as expected and without any limitations\n    - ICMP packets directed to pass through a remote network (e.g. pod to external internet communication via ping) cannot be transposed and thus will *not* be routed back to their source\n      - Since TCP/UDP packets can still be transposed, one can substitute `ping \u003cdestination\u003e` with `curl \u003cdestination\u003e` to be able to debug connectivity to the outside world.\n\n### Windows Container Compatibility\nAs noted above, there are compatibility issues enforced by Microsoft where the host OS version must match the container base image OS. Changes to this compatibility policy must come from Microsoft. For GA, since we will only support Windows Server 2019 (aka 1809), both `container host OS` and `container OS` must be running the same version of Windows, 1809. \n\nHaving said that, a customer can deploy Kubernetes v1.14 with Windows 1809.\n- We will support Windows 1809 with at least 2 additional Kuberneres minor releases (v1.15 and v.1.16)\n- It is possible additional Windows releases (for example Windows 1903) will be added to the support matrix of future Kubernetes releases and they will also be supported for the next 2 versions of Kubernetes after their initial support is announced\n- SIG-Windows will announce support for new Windows operating systems at most twice per year, based on Microsoft's published release cycle\n\nKubernetes minor releases are only supported for 9 months (https://kubernetes.io/docs/setup/version-skew-policy/), which is a smaller support interval than the support interval for Windows bi-annual releases (https://docs.microsoft.com/en-us/windows-server/get-started/windows-server-release-info) \n\nWe don't expect all Windows customers to update the operating system for their apps twice a year. Upgrading your applications is what will dictate and necessitate upgrading or introducing new nodes to the cluster. For the customers that chose to upgrade their operating system for containers running on Kubernetes, we will offer guidance and step-by-step instructions when we add support for a new operating system version. This guidance will include recommended upgrade procedures for upgrading user applications together with cluster nodes.\n\nWindows nodes will adhere to Kubernetes version-skew policy (node to control plane versioning) the same way as Linux nodes do today (https://kubernetes.io/docs/setup/version-skew-policy/)\n\n### Relevant resources/conversations\n\n- [sig-architecture thread](https://groups.google.com/forum/#!topic/kubernetes-sig-architecture/G2zKJ7QK22E)\n- [cncf-k8s-conformance thread](https://lists.cncf.io/g/cncf-k8s-conformance/topic/windows_conformance_tests/27913232)\n- [kubernetes/enhancements proposal](https://github.com/kubernetes/features/issues/116)\n\n\n### Risks and Mitigations\n\n**Second class support**: Kubernetes contributors are likely to be thinking of Linux-based solutions to problems, as Linux remains the primary OS supported. Keeping Windows support working will be an ongoing burden potentially limiting the pace of development. \n\n**User experience**: Users today will need to use some combination of taints and node selectors in order to keep Linux and Windows workloads separated. In the best case this imposes a burden only on Windows users, but this is still less than ideal. The recommended approach is outlined below, with one of its main goals being that we should not break compatibility for existing Linux workloads\n\n#### Ensuring OS-specific workloads land on appropriate container host\nAs you can see below, we plan to document how Windows containers can be scheduled on the appropriate host using Taints and Tolerations. All nodes today have the following default labels (These labels will be graduating to stable soon)\n- beta.kubernetes.io/os = [windows|linux]\n- beta.kubernetes.io/arch = [amd64|arm64|...]\n\nIf a deployment does not specify a nodeSelector like `\"beta.kubernetes.io/os\": windows`, it is possible the Pods can be scheduled on any host, Windows or Linux. This can be problematic since a Windows container can only run on Windows and a Linux container can only run on Linux. The best practice we will recommend is to use a nodeSelector. \n\nHowever, we understand that in many cases customers have a pre-existing large number of deployments for Linux containers, as well as an ecosystem of off-the-shelf configurations, such as community Helm charts, and programmatic pod generation cases, such as with Operators. Customers will be hesitant to make the configuration change to add nodeSelectors. Our proposal as an alternative is to use Taints. Because the kubelet can set Taints during registration, it could easily be modified to automatically add a taint when running on Windows only (`--register-with-taints='os=Win1809:NoSchedule'`). By adding a taint to all Windows nodes, nothing will be scheduled on them (that includes existing Linux Pods). In order for a Windows Pod to be scheduled on a Windows node, it would need both the nodeSelector to choose Windows, and a toleration.\n```\nnodeSelector:\n    \"beta.kubernetes.io/os\": windows\ntolerations:\n    - key: \"os\"\n      operator: \"Equal\"\n      value: \"Win1809\"\n      effect: \"NoSchedule\"\n```\n\n#### Memory Overprovisioning\n\nWindows always treats all user-mode memory allocations as virtual, and pagefiles are mandatory. The net effect is that Windows won't reach out of memory conditions the same way Linux does, and processes will page to disk instead of being subject to out of memory (OOM) termination. There is no way to guarantee a physical memory allocation or reserve for a process - only limits. See [#73417](https://github.com/kubernetes/kubernetes/issues/73417) for more details on the investigation for 1.14.\n\nKeeping memory usage within reasonable bounds is possible with a two-step process. First, use the kubelet parameters `--kubelet-reserve` and/or `--system-reserve` to account for memory usage on the node (outside of containers). This will reduce [NodeAllocatable](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#node-allocatable). As you deploy workloads, use resource limits and reserves on containers. This will also subtract from NodeAllocatable and prevent the scheduler from adding more pods once a node is full. These will be documented as best practices for v1.14. The related kubelet parameters `--eviction-hard`, `--eviction-soft`, and `--enforce-node-allocatable` are invalid for v1.14.\n\nFor later releases, we can work on a configurable heuristic to detect memory pressure, report it through the kubelet `MemoryPressure` condition, and implement pod eviction. \n\n## Graduation Criteria\n- All features and functionality under `What works today` is fully tested and vetted to be working by SIG-Windows\n- SIG-Windows has high confidence to the stability and reliability of Windows Server containers on Kubernetes\n- 100% green/passing conformance tests that are applicable to Windows (see the Testing Plan section for details on these tests). These tests are adequate, non flaky, and continuously run. The test results are publicly accessible, enabled as part of the release-blocking suite\n- Compatibility will not be broken, either for existing users/clusters/features or for the new features going forward, and we will adhere to the deprecation policy (https://kubernetes.io/docs/reference/using-api/deprecation-policy/).\n- Comprehensive documentation that includes but is not limited to the following sections. Documentation will reside at https://kubernetes.io/docs and will adequately cover end user and admin documentation that describes what the user does and how to use it. Not all of the documentation will be under the Getting Started Guide for Windows. Part of it will reside in its own sections (like the Group Managed Service Accounts) and part will be in the Contributor development guide (like the instructions on how to build your own source code)\n1. Outline of Windows Server containers on Kubernetes\n2. Getting Started Guide, including Prerequisites\n3. How to deploy Windows nodes in Kubernetes and where to find the proper binaries (Listed under the changelog for every release. For example https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.13.md#server-binaries)\n4. Overview of Networking on Windows\n5. Links to documentation on how to deploy and use CNI plugins for Windows (example for OVN - https://github.com/openvswitch/ovn-kubernetes/tree/master/contrib)\n6. Links to documentation on how to deploy Windows nodes for public cloud providers or other Kubernetes distributions (example for Rancher - https://rancher.com/docs//rancher/v2.x/en/cluster-provisioning/rke-clusters/windows-clusters/)\n7. How to schedule Windows Server containers, including examples\n8. How to use metrics and the Horizontal Pod Autoscaler\n9. How to use Group Managed Service Accounts\n10. How to use Taints and Tolerations for a heterogeneous compute cluster (Windows + Linux)\n11. How to use Hyper-V isolation (not a stable feature yet)\n12. How to build Kubernetes for Windows from source\n13. Supported functionality (with examples where appropriate)\n14. Known Limitations\n15. Unsupported functionality\n16. Resources for contributing and getting help - Includes troubleshooting help and links to additional troubleshooting guides like https://docs.microsoft.com/en-us/virtualization/windowscontainers/kubernetes/common-problems\n\n## Implementation History\n- Alpha was released with Kubernetes v.1.5\n- Beta was released with Kubernetes v.1.9\n\n## Testing Plan\n\n\n### Test Dashboard\n\nAll test cases will be built in kubernetes/test/e2e, scheduled through [prow](https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes-sigs/sig-windows/sig-windows-config.yaml), and published on the [TestGrid SIG-Windows dashboard](https://testgrid.k8s.io/sig-windows#aks-engine-azure-windows-master) daily. This will be the master list of what needs to pass to be declared stable and will include all tests tagged [SIG-Windows] along with the subset of conformance tests that can pass on Windows. \n\nAdditional dashboard pages will be added over time as we run the same test cases with additional CRI, CNI and cloud providers. They are running the same test cases, and are not required for v1.14 graduation to stable.\n\n- [Windows Server 2019 on GCE](https://testgrid.k8s.io/sig-windows#gce-windows-master)\n- [Windows Server 2019 with Flannel on vSphere](https://testgrid.k8s.io/sig-windows#cfcr-vsphere-windows-master)\n- Windows Server 2019 with OVN+OVS \u0026 Dockershim\n- Windows Server 2019 with OVN+OVS \u0026 CRI-ContainerD\n- Windows Server 2019 with Azure-CNI \u0026 CRI-ContainerD\n- Windows Server 2019 with Flannel \u0026 CRI-ContainerD\n\n### Test Environment\n\nThe primary test environment deployed by [kubetest](https://github.com/kubernetes/test-infra/blob/72c720f29cb43d923ac76b10d25a62c29662683d/kubetest/azure.go#L180) for v1.14 is a group of VMs deployed on Azure:\n\n- 1 Master VM running Ubuntu, size \"Standard_D2s_v3\"\n  - Moby v3.0.1 (https://packages.microsoft.com/ubuntu/16.04/prod/pool/main/m/moby-engine/)\n  - Azure CNI\n- 3 Windows nodes running Windows Server 2019, size \"Standard_D2s_v3\"\n  - Docker EE-Basic v.18.09\n  - Azure CNI\n\nKubetest uses [aks-engine](https://github.com/Azure/aks-engine) to create the deployment template for each of those VMs that's passed on to Azure for deployment. Once the test pass is complete, kubetest deletes the cluster. The Azure subscription used for this test pass is managed by Lachie Evenson \u0026 Patrick Lang. The credentials needed were given to the k8s-infra-oncall team.\n\n### Test Approach\n\nThe testing for Windows nodes will include multiple approaches:\n\n1. [Adapting](#Adapting-existing-tests) some of the existing conformance tests to be able to pass on multiple node OS's. Tests that won't work will be [excluded](https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes-sigs/sig-windows/sig-windows-config.yaml#L69).\n2. Adding [substitute](#Substitute-test-cases) test cases where the first approach isn't feasible or would change the tests in a way is not approved by the owner. These will be tagged with `[SIG-Windows]`\n3. Last, gaps will be filled with [Windows specific tests](#Windows-specific-tests). These will also be tagged with `[SIG-Windows]`\n\nAll of the test cases will be maintained within the kubernetes/kubernetes repo. SIG-Windows specific tests for 2/3 will be in [test/e2e/windows](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/windows)\n\nAdditional Windows test setup scripts, container image source code, and documentation will be kept in the [kubernetes-sigs/windows-testing](https://github.com/kubernetes-sigs/windows-testing) repo. One example is that the prow jobs need a list of repos to use for the test containers, and that will be maintained here - see [windows-testing#1](https://github.com/kubernetes-sigs/windows-testing/issues/1).\nBuilding these containers for Windows requires a Windows build machine, which isn't part of the Kubernetes PR or official builds. If the SIG is given access to a suitable GCR.io account, images can be pushed there. Otherwise, we'll use continue pushing to Docker Hub.\n\n\n#### Adapting existing tests\n\nOver the course of v1.12/13, many conformance tests were adapted to be able to pass on either Linux or Windows nodes as long as matching OS containers are run. This was done by creating Windows equivalent containers from [kubernetes/test/images](https://github.com/kubernetes/kubernetes/tree/master/test/images). An additional parameter is needed for e2e.test/kubetest to change the container repos to the one containing Windows versions since they're not part of the Kubernetes build process yet.\n\nThese tests are already running and listed on the dashboard above, with a few exceptions:\n\n- [x] \"... should function for node-pod communication: udp\" - issue [#72917](https://github.com/kubernetes/kubernetes/issues/72917) has a PR open\n- [x] \"should be able to pull image from docker hub\" - [PR #72777](https://github.com/kubernetes/kubernetes/pull/72777) open\n- [x] \"should provide DNS for the cluster\" - [PR #72729](https://github.com/kubernetes/kubernetes/pull/72729) open for issue [#70189](https://github.com/kubernetes/kubernetes/issues/70189)\n\n\nAnd also some cleanup to simplify the test exclusions:\n - [x] Skip Windows unrelated tests (those are tagged as `LinuxOnly`) - (https://github.com/kubernetes/kubernetes/pull/73204)\n\n#### Substitute test cases\n\nThese are test cases that follow a similar flow to a conformance test that is dependent on Linux-specific functionality, but differs enough that the same test case cannot be used for both Windows \u0026 Linux. Examples include differences in file access permissions (UID/GID vs username, permission octets vs Windows ACLs), and network configuration (`/etc/resolv.conf` is used on Linux, but Windows DNS settings are stored in the Windows registry).\n\nThese test cases are in review:\n\n\n- [x] [sig-network] [sig-windows] Networking Granular Checks: Pods should function for intra-pod communication: http - [PR#71468](https://github.com/kubernetes/kubernetes/pull/71468)\n- [x] [sig-network] [sig-windows] Networking Granular Checks: Pods should function for intra-pod communication: udp - [PR#71468](https://github.com/kubernetes/kubernetes/pull/71468)\n- [x] [sig-network] [sig-windows] Networking Granular Checks: Pods should function for node-pod communication: udp - [PR#71468](https://github.com/kubernetes/kubernetes/pull/71468)\n- [x] [sig-network] [sig-windows] Networking Granular Checks: Pods should function for node-pod communication: http - [PR#71468](https://github.com/kubernetes/kubernetes/pull/71468)\n\n\nAnd these still need to be covered: \n\n- [x] DNS configuration is passed through CNI, not `/etc/resolv.conf` [67435](https://github.com/kubernetes/kubernetes/pull/67435)\n  - Test cases needed for `dnsPolicy`: Default, ClusterFirst, None\n  - Test cases needed for `dnsConfig`\n  - Test cases needed for `hostname`\n  - Test cases needed for `subdomain`\n\nTests will be merged once the CNI plugins are updated. See [Custom DNS updates for CNI plugins](#custom-dns-updates-for-cni-plugins) for full details.\n\n- [x] Windows doesn't have CGroups, but nodeReserve and kubeletReserve are [implemented](https://github.com/kubernetes/kubernetes/pull/69960)\n\n\n\n#### Windows specific tests\n\nWe will also add Windows scenario-specific tests to cover more typical use cases and features specific to Windows. These tests will be in [kubernetes/test/e2e/windows](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/windows). This will also include density and performance tests that are adjusted for Windows apps which have different image sizes and memory requirements.\n\nThese areas still need test cases written:\n\n- [x] System, pod \u0026 network stats are implemented in kubelet, not cadvisor [70121](https://github.com/kubernetes/kubernetes/pull/70121), [66427](https://github.com/kubernetes/kubernetes/pull/66427), [62266](https://github.com/kubernetes/kubernetes/pull/62266), [51152](https://github.com/kubernetes/kubernetes/pull/51152), [50396](https://github.com/kubernetes/kubernetes/pull/50396)\n- [ ] Create a `NodePort` service, and verify it's accessible on both Linux \u0026 Windows node IPs on the correct port [tracked as #73327](https://github.com/kubernetes/kubernetes/issues/73327)\n- [x] Verify `ExternalPort` works from Windows pods [tracked as #73328](https://github.com/kubernetes/kubernetes/issues/73328)\n- [x] Verify `imagePullPolicy` behaviors. The reason behind needing a Windows specific test is because we may need to publish Windows-specific images for this validation. The current tests are pulling Linux images. Long term we will work with the team to use a universal/heterogeneous image if possible.\n\n\n\n## Conformance Testing\n\nThere were lots of discussions with SIG-Architecture and the Conformance working group on what Windows means for conformance. For the purposes of this KEP - graduating Windows node support to stable does not require conformance testing for v1.14, and will be completed later. This also means that clusters with Windows nodes will not be eligible for the conformance logo. During v1.14, SIG-Windows will be finishing the right set of tests so that we can propose changes to existing tests to make them OS agnostic, and what additional Windows-specific tests are needed. With continued work through the conformance working group, our goal would be to move these into a Windows conformance profile for v1.15. This would mean clusters could be tested and certified with only Linux nodes for 1.15+, no different from how they were run in \u003c= 1.14. Windows nodes could be added and tested against the new conformance profile, but not all clusters will require Windows.\n\n\n## API Reference\n\nThis section provides an API by API list of Windows \u0026 Linux differences. Issue [#70604](https://github.com/kubernetes/kubernetes/issues/70604) will be used to track updating the generated API docs with notes on Windows support where needed.\n\nThere are no differences in how most of the Kubernetes APIs work. The subtleties around what's different come down to differences in the OS and container runtime. Where a property on a workload API such as Pod or Container was designed with an assumption that it's implemented on Linux, then that may not hold true on Windows.\n\nAt a high level, these OS concepts are different:\n\n- Identity - Linux uses userID (UID) and groupID (GID) which are represented as integer types. User and group names are not canonical - they are just an alias in /etc/groups or /etc/passwd back to UID+GID. Windows uses a larger binary security identifier (SID) which is stored in the Windows Security Access Manager (SAM) database. This database is not shared between the host and containers, or between containers.\n- File permissions - Windows uses an access control list based on SIDs, rather than a bitmask of permissions and UID+GID\n- File paths - convention on Windows is to use `\\` instead of `/`. The Go IO libraries typically accept both and just make it work, but when you're setting a path or commandline that's interpreted inside a container, `\\` may be needed.\n- Signals - Windows interactive apps handle termination differently, and can implement one or more of these:\n  - A UI thread will handle well-defined messages including [WM_CLOSE](https://docs.microsoft.com/en-us/windows/desktop/winmsg/wm-close)\n  - Console apps will handle `ctrl-c` or `ctrl-break` using a [Control Handler](https://docs.microsoft.com/en-us/windows/console/registering-a-control-handler-function)\n  - Services will register a [Service Control Handler](https://docs.microsoft.com/en-us/windows/desktop/Services/service-control-handler-function) function that can accept `SERVICE_CONTROL_STOP` control codes\n\nThese conventions are the same:\n- Exit Codes mostly follow the same convention where 0 is success, nonzero is failure. The [specific error codes](https://docs.microsoft.com/en-us/windows/desktop/Debug/system-error-codes--0-499-) may differ. Exit codes passed from the Kubernetes components (kubelet, kube-proxy) will be unchanged.\n\n\nThe Windows container runtime also has a few important differences:\n\n- Resource management and process isolation - Linux cgroups are used as a pod boundary for resource controls. Containers are created within that boundary for network, process and filesystem isolation. The cgroups APIs can be used to gather cpu/io/memory stats. Windows uses a Job object per container with a system namespace filter to contain all processes in a container and provide logical isolation from the host.\n  - There is no way to run a Windows container without the namespace filtering in place. This means that system privileges cannot be asserted in the context of the host, and privileged containers are not available on Windows. Containers cannot assume an identity from the host because the SAM is separate.\n- Filesystems - Windows has a layered filesystem driver to mount container layers and create a copy filesystem based on NTFS. All file paths in the container are resolved only within the context of that container.\n  - Volume mounts can only target a directory in the container, and not an individual file.\n  - Volume mounts cannot project files or directories back to the host filesystem.  \n  - Read-only filesystems are not supported because write access is always required for the Windows registry and SAM database. Read-only volumes are supported\n  - Volume user-masks and permissions are not available. Because the SAM is not shared between the host \u0026 container, there's no mapping between them. All permissions are resolved within the context of the container.\n- Networking - The Windows host networking networking service and virtual switch implement namespacing and can create virtual NICs as needed for a pod or container. However, many configurations such as DNS, routes, and metrics are stored in the Windows registry database rather than /etc/... files as they are on Linux. The Windows registry for the container is separate from that of the host, so concepts like mapping /etc/resolv.conf from the host into a container don't have the same effect they would on Linux. These must be configured using Windows APIs run in the context of that container. Therefore CNI implementations need to call  the HNS instead of relying on file mappings to pass network details into the pod or container.\n\n\n\n### V1.Container\n\n- `V1.Container.ResourceRequirements.limits.cpu` and `V1.Container.ResourceRequirements.limits.memory` - Windows doesn't use hard limits for CPU allocations. Instead, a share system is used. The existing fields based on millicores are scaled into relative shares that are followed by the Windows scheduler. [see: kuberuntime/helpers_windows.go](https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/kuberuntime/helpers_windows.go), [see: resource controls in Microsoft docs](https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/resource-controls)\nWhen using Hyper-V isolation (alpha), the hypervisor also needs a number of CPUs assigned. The millicores used in the limit is divided by 1000 to get the number of cores required. The CPU count is a hard limit.\n  - Huge pages are not implemented in the Windows container runtime, and are not available. They require [asserting a user privilege](https://docs.microsoft.com/en-us/windows/desktop/Memory/large-page-support) that's not configurable for containers.\n- `V1.Container.ResourceRequirements.requests.cpu` and `V1.Container.ResourceRequirements.requests.memory` - Requests are subtracted from node available resources, so they can be used to avoid overprovisioning a node. However, they cannot be used to guarantee resources in an overprovisioned node. They should be applied to all containers as a best practice if the operator wants to avoid overprovisioning entirely.\n- `V1.Container.SecurityContext.allowPrivilegeEscalation` - not possible on Windows, none of the capabilies are hooked up\n- `V1.Container.SecurityContext.Capabilities` - POSIX capabilities are not implemented on Windows\n- `V1.Container.SecurityContext.privileged` - Windows doesn't support privileged containers\n- `V1.Container.SecurityContext.procMount` - Windows doesn't have a `/proc` filesystem\n- `V1.Container.SecurityContext.readOnlyRootFilesystem` - not possible on Windows, write access is required for registry \u0026 system processes to run inside the container\n- `V1.Container.SecurityContext.runAsGroup` - not possible on Windows, no GID support\n- `V1.Container.SecurityContext.runAsNonRoot` - Windows does not have a root user. The closest equivalent is `ContainerAdministrator` which is an identity that doesn't exist on the node.\n- `V1.Container.SecurityContext.runAsUser` - not possible on Windows, no UID support as int. This needs to change to IntStr, see [64009](https://github.com/kubernetes/kubernetes/pull/64009), to support Windows users as strings, or another field is needed. Work remaining tracked in [#73387](https://github.com/kubernetes/kubernetes/issues/73387)\n- `V1.Container.SecurityContext.seLinuxOptions` - not possible on Windows, no SELinux\n- `V1.Container.terminationMessagePath` - this has some limitations in that Windows doesn't support mapping single files. The default value is `/dev/termination-log`, which does work because it does not exist on Windows by default.\n\n\n### V1.Pod\n\n- `V1.Pod.hostIPC`, `v1.pod.hostpid` - host namespace sharing is not possible on Windows\n- `V1.Pod.hostNetwork` - There is no Windows OS support to share the host network\n- `V1.Pod.dnsPolicy` - ClusterFirstWithHostNet - is not supported because Host Networking is not supported on Windows.\n- `V1.Pod.podSecurityContext` - see [V1.PodSecurityContext](#v1podsecuritycontext)\n- `V1.Pod.shareProcessNamespace` - this is an beta feature, and depends on Linux namespaces which are not implemented on Windows. Windows cannot share process namespaces or the container's root filesystem. Only the network can be shared.\n- `V1.Pod.terminationGracePeriodSeconds` - this is not fully implemented in Docker on Windows, see: [reference](https://github.com/moby/moby/issues/25982). The behavior today is that the ENTRYPOINT process is sent `CTRL_SHUTDOWN_EVENT`, then Windows waits 5 seconds by hardcoded default, and finally shuts down all processes using the normal Windows shutdown behavior. The 5 second default is actually in the Windows registry [inside the container](https://github.com/moby/moby/issues/25982#issuecomment-426441183), so it can be overridden when the container is built. Runtime configuration will be feasible in CRI-ContainerD but not for v1.14. Issue [#73434](https://github.com/kubernetes/kubernetes/issues/73434) is tracking this for a later release.\n- `V1.Pod.volumeDevices` - this is an beta feature, and is not implemented on Windows. Windows cannot attach raw block devices to pods.\n- `V1.Pod.volumes` - EmptyDir, Secret, ConfigMap, HostPath - all work and have tests in TestGrid\n  - `V1.emptyDirVolumeSource` - the Node default medium is disk on Windows. `memory` is not supported, as Windows does not have a built-in RAM disk.\n- `V1.VolumeMount.mountPropagation` - is not supported on Windows\n\n### V1.PodSecurityContext\n\nNone of the PodSecurityContext fields work on Windows. They're listed here for reference.\n\n- `V1.PodSecurityContext.SELinuxOptions` - SELinux is not available on Windows\n- `V1.PodSecurityContext.RunAsUser` - provides a UID, not available on Windows\n- `V1.PodSecurityContext.RunAsGroup` - provides a GID, not available on Windows\n- `V1.PodSecurityContext.RunAsNonRoot` - Windows does not have a root user. The closest equivalent is `ContainerAdministrator` which is an identity that doesn't exist on the node.\n- `V1.PodSecurityContext.SupplementalGroups` - provides GID, not available on Windows\n- `V1.PodSecurityContext.Sysctls` - these are part of the Linux sysctl interface. There's no equivalent on Windows.\n\n\n## Other references\n\n[Past release proposal for v1.12/13](https://docs.google.com/document/d/1YkLZIYYLMQhxdI2esN5PuTkhQHhO0joNvnbHpW68yg8/edit#)\n"
+  },
+  {
+    "id": "8a267023af73ffe4d107795f9eed7bc4",
+    "title": "Windows security context API changes",
+    "authors": ["@ddebroy"],
+    "owningSig": "sig-windows",
+    "participatingSigs": null,
+    "reviewers": ["@patricklang", "@liggitt"],
+    "approvers": ["@liggitt"],
+    "editor": "TBD",
+    "creationDate": "2019-04-18",
+    "lastUpdated": "2019-04-18",
+    "status": "implementable",
+    "seeAlso": [
+      "keps/sig-windows/20181221-windows-group-managed-service-accounts-for-container-identity.md"
+    ],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Windows specific options in Pod Security Context and Container Security Context\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Specify GMSA credential spec and RunAsUserName for a pod](#specify-gmsa-credential-spec-and-runasusername-for-a-pod)\n    - [Specify distinct GMSA credential spec for a container in a pod (while retaining RunAsUserName from pod spec)](#specify-distinct-gmsa-credential-spec-for-a-container-in-a-pod-while-retaining-runasusername-from-pod-spec)\n  - [Implementation Details/Notes/Constraints [optional]](#implementation-detailsnotesconstraints-optional)\n    - [Validation of fields](#validation-of-fields)\n    - [Specification of both GMSA credspec and RunAsUserName](#specification-of-both-gmsa-credspec-and-runasusername)\n    - [Changes in kubelet](#changes-in-kubelet)\n  - [Risks and Mitigations](#risks-and-mitigations)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade and Version Skew Strategy](#upgrade--downgrade-and-version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Implementation Roadmap](#implementation-roadmap)\n- [Drawbacks [optional]](#drawbacks-optional)\n- [Alternatives [optional]](#alternatives-optional)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nIn this KEP, we propose API enhancements in the Kubernetes pod spec to capture Windows OS specific security options from the perspective of Windows workload identity in containers. Initially the enhancements will cover fields pertinent to GMSA credential specs and the username with which to execute the container entry-point. More fields may be added in the future. Please note that this is a KEP with a very limited scope focussing mainly on the API enhancements needed to support GMSA credential spec details and the RunAsUsername field. Details around overall GMSA functionality can be found in the [GMSA](https://github.com/kubernetes/enhancements/blob/master/keps/sig-windows/20181221-windows-group-managed-service-accounts-for-container-identity.md) KEP and are not repeated in this KEP.\n\n## Motivation\n\n There are two important motivations for the API enhancements described in this KEP:\n \n 1. With the introduction of Alpha support for GMSA in Kubernetes v1.14, references to GMSA credential spec custom resources need to be specified through annotations at the pod and container level. However, as detailed in the related [GMSA](https://github.com/kubernetes/enhancements/blob/master/keps/sig-windows/20181221-windows-group-managed-service-accounts-for-container-identity.md) KEP, we want the ability to specify references to GMSA credential specs directly in the pod/container specs as fields (without having to use annotations) beyond the Alpha stage.\n \n 2. The Windows implementation of the dockershim, CRI and the low-level OCI spec can already handle a username (instead of UID), which is interpreted inside the container to create a process as the intended user. This however is not surfaced as a field in the pod/container specs that an operator can specify. We want the ability to specify the desired username in the pod/container specs as fields and be able to pass them to the configured Windows runtime.\n\n### Goals\n\nPropose API enhancements in existing `PodSecurityContext` and `SecurityContext` structs for pods and individual containers respectively to allow operators to specify:\n- Name of a GMSA credential spec custom resource \n- Full GMSA credential spec JSON\n- A Windows username whose identity will be used to kick off the entrypoint process in containers.\n\n### Non-Goals\n\n- Details around GMSA end-2-end functionality and interaction with webhooks that is covered in the [GMSA](https://github.com/kubernetes/enhancements/blob/master/keps/sig-windows/20181221-windows-group-managed-service-accounts-for-container-identity.md) KEP.\n- Implementation details and security considerations around how a GMSACredentialSpecName is expanded to GMSACredentialSpec as that is covered in details in the [GMSA](https://github.com/kubernetes/enhancements/blob/master/keps/sig-windows/20181221-windows-group-managed-service-accounts-for-container-identity.md) KEP.\n- Details around how GMSA credential specs or Windows username is passed through CRI and interpreted by Windows container run-times like Docker or ContainerD. Enhancements related to GMSA in CRI is already covered in the [GMSA](https://github.com/kubernetes/enhancements/blob/master/keps/sig-windows/20181221-windows-group-managed-service-accounts-for-container-identity.md) KEP. Enhancements related to Username in CRI was introduced a while back in a [PR](https://github.com/kubernetes/kubernetes/pull/64009) \n\n## Proposal\n\nIn this KEP we propose a new field named `WindowsOptions` in the `PodSecurityContext` struct (associated with a pod spec) and the `SecurityContext` struct (associated with each container in a pod). `WindowsOptions` will be a pointer to a new struct of type `WindowsSecurityContextOptions`. The `WindowsOptions` struct will contain Windows OS specific security attributes scoped either at the pod level (applicable to all containers in the pod) or at the individual container level (that can override the pod level specification). This is inspired by the existing `SELinuxOptions` field that groups Linux specific SELinux options. Initially, the `WindowsSecurityContextOptions` struct will have the following fields:\n\n- GMSACredentialSpecName: A string specifying the name of a GMSA credential spec custom resource\n- GMSACredentialSpec: A string specifying the full credential spec JSON string associated with a GMSA credential spec\n- RunAsUserName: A string specifying the user name in Windows to run the entrypoint of the container\n\nMore fields may be added to the `WindowsSecurityContextOptions` struct as desired in the future.\n\nNote that the GMSACredentialSpec will almost never be populated by operators. It will be auto populated by a webhook that will look up the credential spec JSON associated with GMSACredentialSpecName and expand it. Details of the interactions with the webhook is covered in [GMSA](https://github.com/kubernetes/enhancements/blob/master/keps/sig-windows/20181221-windows-group-managed-service-accounts-for-container-identity.md) KEP\n\n### User Stories\n\nThere could be various ways in which an operator may specify individual members of `WindowsOptions` in a pod spec scoped either at the pod level or at an individual container level.\n\n#### Specify GMSA credential spec and RunAsUserName for a pod\n\n```\napiVersion: v1\nkind: Pod\nmetadata:\n  name: iis\n  labels:\n    name: iis\nspec:\n  securityContext:\n    windowsOptions:\n      gmsaCredentialSpecName: webapp1-credspec\n      runAsUserName: \"NT AUTHORITY\\\\NETWORK SERVICE\"\n  containers:\n    - name: iis\n      image: mcr.microsoft.com/windows/servercore/iis:windowsservercore-ltsc2019\n      ports:\n        - containerPort: 80\n    - name: logger\n      image: eventlogger:2019\n      ports:\n        - containerPort: 80\n  nodeSelector:\n    beta.kubernetes.io/os : windows\n```\n\n#### Specify distinct GMSA credential spec for a container in a pod (while retaining RunAsUserName from pod spec)\n\n```\napiVersion: v1\nkind: Pod\nmetadata:\n  name: iis\n  labels:\n    name: iis\nspec:\n  securityContext:\n    windowsOptions:\n      gmsaCredentialSpecName: webapp1-credspec\n      runAsUserName: \"NT AUTHORITY\\\\NETWORK SERVICE\"\n  containers:\n    - name: iis\n      image: mcr.microsoft.com/windows/servercore/iis:windowsservercore-ltsc2019\n      ports:\n        - containerPort: 80\n    - name: logger\n      securityContext:\n        windowsOptions:\n          gmsaCredentialSpecName: eventlogger-credspec\n      image: eventlogger:2019\n      ports:\n        - containerPort: 80\n  nodeSelector:\n    beta.kubernetes.io/os : windows\n```\n\n### Implementation Details/Notes/Constraints [optional]\n\nThe `WindowsSecurityContextOptions` struct will be defined as follows in pkg/apis/core/types.go\n\n```\n// WindowsSecurityContextOptions contain Windows-specific options and credentials.\ntype WindowsSecurityContextOptions struct {\n\t// GMSACredentialSpecName is the name of the GMSA credential spec to use.\n\t// +optional\n\tGMSACredentialSpecName string\n\n\t// GMSACredentialSpec is where the GMSA admission webhook\n\t// (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the\n\t// GMSA credential spec named by the `GmsaCredentialSpecName` field.\n\t// +optional\n\tGMSACredentialSpec string\n\n\t// RunAsUserName is the local user context used to log in to the container\n\t// +optional\n\tRunAsUserName string\n}\n```\n\nField `WindowsOptions *WindowsSecurityContextOptions` will be added to `SecurityContext` and `PodSecurityContext` structs.\n\n#### Validation of fields\n\nThe fields within the `WindowsSecurityContextOptions` struct will be validated as follows:\n- GMSACredentialSpecName: This field will be the name of a custom resource. It should follow the rules associated with [naming of resources](https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names) in Kubernetes. Validation will make sure the maximum number of characters is 253  and consist of lower case alphanumeric characters, -, and .\n- GMSACredentialSpec: The size of the CredentialSpec JSON blob should be limited to avoid abuse. We will limit it to 64K which allows for a lot of room to specify extremely complicated credential specs. Typically this JSON blob is not expected to be more than 1K based on experience so far.\n- RunAsUserName: This field needs to allow for a valid set of usernames allowed for Windows containers. Currently the OCI spec or Windows documentation does not specify any clear restrictions around the length of this parameter or restrictions around usage of special characters when passed to the container runtime and eventually to Windows HCS. So Kubernetes validation won't enforce any character set validation. A maximum length of 256 characters will be allowed to prevent abuse.\n\nAt this time, no additional API validation logic will be implemented on the pod spec's existing security context structs to check `SELinuxOptions` or other Linux specific fields like `RunAsUser`/`RunAsGroup` cannot be specified along with `WindowsOptions` within the same security context. If such validation and fast fail behavior is desired and external admission controller can implement the additional checks in future.\n\n#### Specification of both GMSA credspec and RunAsUserName\n\nNote that both GMSA credspec and RunAsUserName may be specified. Specification of one field is not mutually exclusive with the other. RunAsUserName governs the local user identity used to log into the container. This is decoupled from the GMSA domain identity used to interact with network resources. To use GMSA identity, processes in the container should run as \"Local System\" or \"Network Service\" users. However Kubernetes won't enforce any rules around specification of these fields. For further details, please refer [here](https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/manage-serviceaccounts#configuring-your-application-to-use-the-gmsa)\n\n#### Changes in kubelet\n\nAn effective value for a field in a container's `SecurityContext.WindowsOptions` will be determined by calling `DetermineEffectiveSecurityContext`. If a value is specified for a field in the pod's `PodSecurityContext.WindowsOptions`, `DetermineEffectiveSecurityContext` will apply it to the corresponding field in a container's `SecurityContext.WindowsOptions` unless that field is already set with a different value in the container's `SecurityContext.WindowsOptions`.\n\n### Risks and Mitigations\n\nNone\n\n## Design Details\n\n### Test Plan\n\nUnit tests will be added around `DetermineEffectiveSecurityContext` to ensure for each field in `WindowsOptions` the values in a container's `SecurityContext.WindowsOptions` can override values of the fields in the pod's `PodSecurityContext.WindowsOptions`.\n\nAn e2e test case with `[sig-windows]` label will be added to the Windows test runs to exercise and verify RunAsUserName is correctly taking effect. The test will try to bring up a pod with a container based on NanoServer, default user `ContainerUser` and the entrypoint trying to execute a privileged operation like changing the routing table inside the container. The pod spec will specify `ContainerAdministrator` as the RunAsUserName to verify the privileged operation within the container can successfully execute. Without RunAsUserName working correctly, the default user in the container will fail to execute the privileged operation and the container will exit prematurely.\n\nOverall test plans for GMSA are covered in the [GMSA](https://github.com/kubernetes/enhancements/blob/master/keps/sig-windows/20181221-windows-group-managed-service-accounts-for-container-identity.md) KEP.\n\n### Graduation Criteria\n\nThe API change introduced in this KEP will not be feature gated in any form but will be considered \"alpha\" fields initially. The code for populating the proposed fields in this KEP will be feature gated in an `alpha` state for at least one release as detailed in the Version Skew Strategy section below.\n\n### Upgrade / Downgrade and Version Skew Strategy\n\nOnly feature gated code will populate the new `WindowsOptions` field in the Kubernetes release that the field is introduced in (e.g. 1.15). If the feature gates are disabled (default configuration), the new fields will be set to empty/zero values during creation and updates. After stabilizing for one release cycle, code paths that populate the `WindowsOptions` field may be enabled by default (e.g. in 1.16). This will adhere with the [published guidelines](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api_changes.md#alpha-field-in-existing-api-version) for introducing \"alpha\" fields in current stable APIs. This rollout plan ensures a smooth upgrade in the context of persistence of data for the new fields in clusters with masters/APIServer in a HA configuration.\n\n## Implementation History\n\nhttps://github.com/kubernetes/kubernetes/pull/73609 introducing RunAsUsername was held back from 1.14 since it was too late for API review. This PR will be rebased and adjusted to meet this KEP's proposed API.\n\n## Implementation Roadmap\n\nHere is the planned sequence of PRs to introduce the API change:\n\nFirst, we will introduce an empty `WindowsSecurityContextOptions` struct and corresponding `WindowsOptions` fields in the `PodSecurityContext` and `SecurityContext` structs.\n\nNext, independent feature oriented PRs [for GMSA and RunAsUsername for now] will introduce the necessary fields in the `WindowsSecurityContextOptions` struct. These PRs need to follow existing guidelines around API spec changes: API changes in an isolated commit, generated changes in an isolated commit, and implementation in one or more commits.\n\nIf development of the features progress in parallel, generated protobuf field IDs may conflict and require appropriate updates and rebasing.\n\n## Drawbacks [optional]\n\n## Alternatives [optional]\n\nThe main alternatives to the API changes is to continue to use annotations as they are done already for GMSA. However, we do not want to continue to rely on annotations for a feature as it matures.\n\nWe considered the option to embed the Windows specific fields directly into the `PodSecurityContext` and `SecurityContext` structs (without grouping them into the `WindowsOptions` struct). However, grouping the fields into OS-specific structs will align more with potential future re-design efforts of the pod spec where there will be structs grouping OS independent fields and OS specific fields. Some of the current Linux fields that do not apply to Windows will not be grouped right away for backward compatibility.\n"
+  },
+  {
+    "id": "cbc6aa2db160564b2744ff6ab7ae64cc",
+    "title": "Supporting CRI-ContainerD on Windows",
+    "authors": ["@patricklang"],
+    "owningSig": "sig-windows",
+    "participatingSigs": ["sig-windows"],
+    "reviewers": ["@yujuhong", "@derekwaynecarr", "@tallclair"],
+    "approvers": ["@michmike"],
+    "editor": "TBD",
+    "creationDate": "2019-04-24",
+    "lastUpdated": "2019-09-20",
+    "status": "implementable",
+    "seeAlso": null,
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# Supporting CRI-ContainerD on Windows\n\n## Table of Contents\n\n\u003c!-- TOC --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Improving Kubernetes integration for Windows Server containers](#improving-kubernetes-integration-for-windows-server-containers)\n    - [Improved isolation and compatibility between Windows pods using Hyper-V](#improved-isolation-and-compatibility-between-windows-pods-using-hyper-v)\n    - [Improve Control over Memory \u0026amp; CPU Resources with Hyper-V](#improve-control-over-memory--cpu-resources-with-hyper-v)\n    - [Improved Storage Control with Hyper-V](#improved-storage-control-with-hyper-v)\n    - [Enable runtime resizing of container resources](#enable-runtime-resizing-of-container-resources)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n    - [Proposal: Use Runtimeclass Scheduler to simplify deployments based on OS version requirements](#proposal-use-runtimeclass-scheduler-to-simplify-deployments-based-on-os-version-requirements)\n    - [Proposal: Standardize hypervisor annotations](#proposal-standardize-hypervisor-annotations)\n  - [Dependencies](#dependencies)\n      - [Windows Server 2019](#windows-server-2019)\n      - [CRI-ContainerD](#cri-containerd)\n      - [CNI: Flannel](#cni-flannel)\n      - [CNI: Kubenet](#cni-kubenet)\n      - [CNI: GCE](#cni-gce)\n      - [Storage: in-tree AzureFile, AzureDisk, Google PD](#storage-in-tree-azurefile-azuredisk-google-pd)\n      - [Storage: FlexVolume for iSCSI \u0026amp; SMB](#storage-flexvolume-for-iscsi--smb)\n  - [Risks and Mitigations](#risks-and-mitigations)\n    - [CRI-ContainerD availability](#cri-containerd-availability)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Alpha release](#alpha-release)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n  - [CRI-O](#cri-o)\n- [Infrastructure Needed](#infrastructure-needed)\n\u003c!-- /TOC --\u003e\n\n## Release Signoff Checklist\n\n**ACTION REQUIRED:** In order to merge code into a release, there must be an issue in [kubernetes/enhancements] referencing this KEP and targeting a release milestone **before [Enhancement Freeze](https://github.com/kubernetes/sig-release/tree/master/releases)\nof the targeted release**.\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e., [kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be updated for the enhancement to be released.\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [x] KEP approvers have set the KEP status to `implementable`\n- [x] Design details are appropriately documented\n- [x] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [x] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nThe ContainerD maintainers have been working on CRI support which is stable on Linux, but is not yet available for Windows as of ContainerD 1.2. Currently it’s planned for ContainerD 1.3, and the developers in the Windows container platform team have most of the key work merged into master already. Supporting CRI-ContainerD on Windows means users will be able to take advantage of the latest container platform improvements that shipped in Windows Server 2019 / 1809 and beyond.\n\n\n## Motivation\n\nWindows Server 2019 includes an updated host container service (HCS v2) that offers more control over how containers are managed. This can remove some limitations and improve some Kubernetes API compatibility. However, the current Docker EE 18.09 release has not been updated to work with the Windows HCSv2, only ContainerD has been migrated. Moving to CRI-ContainerD allows the Windows OS team and Kubernetes developers to focus on an interface designed to work with Kubernetes to improve compatibility and accelerate development.\n\nAdditionally, users could choose to run with only CRI-ContainerD instead of Docker EE if they wanted to reduce the install footprint or produce their own self-supported CRI-ContainerD builds.\n\n### Goals\n\n- Improve the matrix of Kubernetes features that can be supported on Windows\n- Provide a path forward to implement Kubernetes-specific features that are not available in the Docker API today\n- Align with `dockershim` deprecation timelines once they are defined\n\n### Non-Goals\n\n- Running Linux containers on Windows nodes. This would be addressed as a separate KEP since the use cases are different.\n- Deprecating `dockershim`. This is out of scope for this KEP. The effort to migrate that code out of tree is in [KEP PR 866](https://github.com/kubernetes/enhancements/pull/866) and deprecation discussions will happen later.\n\n## Proposal\n\n### User Stories\n\n#### Improving Kubernetes integration for Windows Server containers\n\nMoving to the new Windows HCSv2 platform and ContainerD would allow Kubernetes to add support for:\n\n- Mounting single files, not just folders, into containers\n- Termination messages (depends on single file mounts)\n- /etc/hosts (c:\\windows\\system32\\drivers\\etc\\hosts) file mapping\n\n#### Improved isolation and compatibility between Windows pods using Hyper-V \n\nHyper-V enables each pod to run within it’s own hypervisor partition, with a separate kernel. This means that we can build forward-compatibility for containers across Windows OS versions - for example a container built using Windows Server 1809, could be run on a node running Windows Server 1903. This pod would use the Windows Server 1809 kernel to preserve full compatibility, and other pods could run using either a shared kernel with the node, or their own isolated Windows Server 1903 kernels. Containers requiring 1809 and 1903 (or later) cannot be mixed in the same pod, they must be deployed in separate pods so the matching kernel may be used. Running Windows Server version 1903 containers on a Windows Server 2019/1809 host will not work.\n\nIn addition, some customers may desire hypervisor-based isolation as an additional line of defense against a container break-out attack.\n\nAdding Hyper-V support would use [RuntimeClass](https://kubernetes.io/docs/concepts/containers/runtime-class/#runtime-class). \n3 typical RuntimeClass names would be configured in CRI-ContainerD to support common deployments:\n- runhcs-wcow-process [default] - process isolation is used, container \u0026 node OS version must match\n- runhcs-wcow-hypervisor - Hyper-V isolation is used, Pod will be compatible with containers built with Windows Server 2019 / 1809. Physical memory overcommit is allowed with overages filled from pagefile.\n- runhcs-wcow-hypervisor-1903 - Hyper-V isolation is used, Pod will be compatible with containers built with Windows Server 1903. Physical memory overcommit is allowed with overages filled from pagefile.\n\nUsing Hyper-V isolation does require some extra memory for the isolated kernel \u0026 system processes. This could be accounted for by implementing the [PodOverhead](https://kubernetes.io/docs/concepts/containers/runtime-class/#runtime-class) proposal for those runtime classes. We would include a recommended PodOverhead in the default CRDs, likely between 100-200M.\n\n\n#### Improve Control over Memory \u0026 CPU Resources with Hyper-V\n\nThe Windows kernel itself cannot provide reserved memory for pods, containers or processes. They are always fulfilled using virtual allocations which could be paged out later. However, using a Hyper-V partition improves control over memory and CPU cores. Hyper-V can either allocate memory on-demand (while still enforcing a hard limit), or it can be reserved as a physical allocation up front. Physical allocations may be able to enable large page allocations within that range (to be confirmed) and improve cache coherency. CPU core counts may also be limited so a pod only has certain cores available, rather than shares spread across all cores, and applications can tune thread counts to the actually available cores.\n\nOperators could deploy additional RuntimeClasses with more granular control for performance critical workloads:\n- 2019-Hyper-V-Reserve: Hyper-V isolation is used, Pod will be compatible with containers built with Windows Server 2019 / 1809. Memory reserve == limit, and is guaranteed to not page out.\n  - 2019-Hyper-V-Reserve-\u003cN\u003eCore: Same as above, except all but \u003cN\u003e CPU cores are masked out.\n- 1903-Hyper-V-Reserve: Hyper-V isolation is used, Pod will be compatible with containers built with Windows Server 1903. Memory reserve == limit, and is guaranteed to not page out.\n  - 1903-Hyper-V-Reserve-\u003cN\u003eCore: Same as above, except all but \u003cN\u003e CPU cores are masked out.\n\n\n#### Improved Storage Control with Hyper-V\n\n\nHyper-V also brings the capability to attach storage to pods using block-based protocols (SCSI) instead of file-based protocols (host file mapping / NFS / SMB). These capabilities could be enabled in HCSv2 with CRI-ContainerD, so this could be an area of future work. Some examples could include:\n\nAttaching a \"physical disk\" (such as a local SSD, iSCSI target, Azure Disk or Google Persistent Disk) directly to a pod. The kubelet would need to identify the disk beforehand, then attach it as the pod is created with CRI. It could then be formatted and used within the pod without being mounted or accessible on the host.\n\nCreating [Persistent Local Volumes](https://kubernetes.io/docs/concepts/storage/volumes/#local) using a local virtual disk attached directly to a pod. This would create local, non-resilient storage that could be formatted from the pod without being mounted on the host. This could be used to build out more resource controls such as fixed disk sizes and QoS based on IOPs or throughput and take advantage of high speed local storage such as temporary SSDs offered by cloud providers.\n\n\n#### Enable runtime resizing of container resources\n\nWith virtual-based allocations and Hyper-V, it should be possible to increase the limit for a running pod. This won’t give it a guaranteed allocation, but will allow it to grow without terminating and scheduling a new pod. This could be a path to vertical pod autoscaling. This still needs more investigation and is mentioned as a future possibility.\n\n\n### Implementation Details/Notes/Constraints\n\nThe work needed will span multiple repos, SIG-Windows will be maintaining a [Windows CRI-Containerd Project Board] to track everything in one place.\n\n#### Proposal: Use Runtimeclass Scheduler to simplify deployments based on OS version requirements\n\nAs of version 1.14, RuntimeClass is not considered by the Kubernetes scheduler. There’s no guarantee that a node can start a pod, and it could fail until it’s scheduled on an appropriate node. Additional node labels and nodeSelectors are required to avoid this problem. [RuntimeClass Scheduling](https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/runtime-class-scheduling.md) proposes being able to add nodeSelectors automatically when using a RuntimeClass, simplifying the deployment.\n\nWindows forward compatibility will bring a new challenge as well because there are two ways a container could be run:\n- Constrained to the OS version it was designed for, using process-based isolation\n- Running on a newer OS version using Hyper-V.\nThis second case could be enabled with a RuntimeClass. If a separate RuntimeClass was used based on OS version, this means the scheduler could find a node with matching class.\n\n#### Proposal: Standardize hypervisor annotations\n\nThere are large number of [Windows annotations](https://github.com/Microsoft/hcsshim/blob/master/internal/oci/uvm.go#L15) defined that can control how Hyper-V will configure its hypervisor partition for the pod. Today, these could be set in the runtimeclasses defined in the CRI-ContainerD configuration file on the node, but it would be easier to maintain them if key settings around resources (cpu+memory+storage) could be aligned across multiple hypervisors and exposed in CRI.\n\nDoing this would make pod definitions more portable between different isolation types. It would also avoid the need for a \"t-shirt size\" list of RuntimeClass instances to choose from:\n- 1809-Hyper-V-Reserve-2Core-PhysicalMemory\n- 1903-Hyper-V-Reserve-1Core-VirtualMemory\n- 1903-Hyper-V-Reserve-4Core-PhysicalMemory\n...\n\n### Dependencies\n\n##### Windows Server 2019\n\nThis work would be carried out and tested using the already-released Windows Server 2019. That will enable customers a migration path from Docker 18.09 to CRI-ContainerD if they want to get this new functionality. Windows Server 1903 and later will also be supported once they’re tested.\n\n##### CRI-ContainerD\n\nIt was announced that the upcoming 1.3 release would include Windows support, but that release and timeline are still in planning as of early April 2019.\n\nThe code needed to run ContainerD is merged, and [experimental support in moby](https://github.com/moby/moby/pull/38541) has merged. CRI is in the process of being updated, and open issues are tracked on the [Windows CRI-Containerd Project Board]\n\nThe CRI plugin changes needed to enable Hyper-V isolation are still in a development branch [jterry75/cri](https://github.com/jterry75/cri/tree/windows_port/cmd/containerd) and don’t have an upstream PR open yet.\n\nCode: mostly done\nCI+CD: lacking\n\n##### CNI: Flannel \nFlannel isn’t expected to require any changes since the Windows-specific metaplugins ship outside of the main repo. However, there is still not a stable release supporting Windows so it needs to be built from source. Additionally, the Windows-specific metaplugins to support ContainerD are being developed in a new repo [Microsoft/windows-container-networking](https://github.com/Microsoft/windows-container-networking). It’s still TBD whether this code will be merged into [containernetworking/plugins](https://github.com/containernetworking/plugins/), or maintained in a separate repo.\n- Sdnbridge - this works with host-gw mode, replaces win-bridge\n- Sdnoverlay - this works with vxlan overlay mode, replaces win-overlay\n\nCode: in progress\nCI+CD: lacking\n\n##### CNI: Kubenet\n\nThe same sdnbridge plugin should work with kubenet as well. If someone would like to use kubenet instead of flannel, that should be feasible.\n\n##### CNI: GCE\n\nGCE uses the win-bridge meta-plugin today for managing Windows network interfaces. This would also need to migrate to sdnbridge.\n\n##### Storage: in-tree AzureFile, AzureDisk, Google PD\n\nThese are expected to work and the same tests will be run for both dockershim and CRI-ContainerD.\n\n##### Storage: FlexVolume for iSCSI \u0026 SMB\nThese out-of-tree plugins are expected to work, and are not tested in prow jobs today. If they graduate to stable we’ll add them to testgrid.\n\n### Risks and Mitigations\n\n#### CRI-ContainerD availability\n\nAs mentioned earlier, builds are not yet available. We will publish the setup steps required to build \u0026 test in the kubernetes-sigs/windows-testing repo during the course of alpha so testing can commence.\n\n## Design Details\n\n### Test Plan\n\nThe existing test cases running on Testgrid that cover Windows Server 2019 with Docker will be reused with CRI-ContainerD. Testgrid will include results for both ContainerD and dockershim.\n\n- TestGrid: SIG-Windows: [flannel-l2bridge-windows-master](https://testgrid.k8s.io/sig-windows#flannel-l2bridge-windows-master) - this uses dockershim\n- TestGrid: SIG-Windows: [containerd-l2bridge-windows-master](https://testgrid.k8s.io/sig-windows#containerd-l2bridge-windows-master) - this uses ContainerD\n\nTest cases that depend on ContainerD and won't pass with Dockershim will be marked with `[feature:windows-containerd]` until `dockershim` is deprecated.\n\n### Graduation Criteria\n\n### Alpha release\n\n\u003e Proposed for 1.18\n\n- Windows Server 2019 containers can run with process level isolation\n- TestGrid has results for Kubernetes master branch. CRI-ContainerD and CNI built from source and may include non-upstream PRs.\n\n### Alpha -\u003e Beta Graduation\n\n\u003e Proposed for 1.19 or later\n\n- Feature parity with dockershim, including:\n  - Group Managed Service Account support\n  - Named pipe \u0026 Unix domain socket mounts\n- Support RuntimeClass to enable Hyper-V isolation\n- Publically available builds (beta or better) of CRI-ContainerD, at least one CNI\n- TestGrid results for above builds with Kubernetes master branch\n\n##### Beta -\u003e GA Graduation\n\n\u003e Proposed for 1.20 or later\n\n- Stable release of CRI-ContainerD on Windows, at least one CNI\n- Master \u0026 release branches on TestGrid\n\n### Upgrade / Downgrade Strategy\n\nBecause no Kubernetes API changes are expected, there is no planned upgrade/downgrade testing at the cluster level.\n\nNode upgrade/downgrade is currently out of scope of the Kubernetes project, but we'll aim to include CRI-ContainerD in other efforts such as `kubeadm` bootstrapping for nodes.\n\nAs discussed in SIG-Node, there's also no testing on switching CRI on an existing node. These are expected to be installed and configured as a prerequisite before joining a node to the cluster.\n\n### Version Skew Strategy\n\nThere's no version skew considerations needed for the same reasons described in upgrade/downgrade strategy.\n\n## Implementation History\n\n- 2019-04-24 - KEP started, based on the [earlier doc shared SIG-Windows and SIG-Node](https://docs.google.com/document/d/1NigFz1nxI9XOi6sGblp_1m-rG9Ne6ELUrNO0V_TJqhI/edit)\n- 2019-09-20 - Updated with new milestones\n- 2020-01-21 - Updated with new milestones\n\n## Alternatives\n\n### CRI-O\n\n[CRI-O](https://cri-o.io/) is another runtime that aims to closely support all the fields available in the CRI spec. Currently there aren't any maintainers porting it to Windows so it's not a viable alternative.\n\n## Infrastructure Needed\n\nNo new infrastructure is currently needed from the Kubernetes community. The existing test jobs using prow \u0026 testgrid will be copied and modified to test CRI-ContainerD in addition to dockershim.\n\n\n\n[Windows CRI-Containerd Project Board]: https://github.com/orgs/kubernetes/projects/34\n"
+  },
+  {
+    "id": "2805f42b52ed0ae79b0e3d02f407cc78",
+    "title": "Support for CSI Plugins on Windows Nodes",
+    "authors": ["@ddebroy"],
+    "owningSig": "sig-windows",
+    "participatingSigs": ["sig-windows", "sig-storage"],
+    "reviewers": ["@patricklang", "@michmike", "@jingxu97", "@yujuhong", "@msau42"],
+    "approvers": ["@patricklang", "@msau42"],
+    "editor": "TBD",
+    "creationDate": "2019-07-14",
+    "lastUpdated": "2019-07-29",
+    "status": "implementable",
+    "seeAlso": ["NA"],
+    "replaces": ["NA"],
+    "supersededBy": ["NA"],
+    "markdown": "\n# Support for CSI Plugins on Windows Nodes\n\n## Table of Contents\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n    - [Deploy CSI Node Plugin DaemonSet targeting Windows nodes](#deploy-csi-node-plugin-daemonset-targeting-windows-nodes)\n    - [Deploy Windows workloads that consume persistent storage managed by a CSI plugin](#deploy-windows-workloads-that-consume-persistent-storage-managed-by-a-csi-plugin)\n  - [Implementation Details](#implementation-details)\n    - [Enhancements in Kubelet Plugin Watcher](#enhancements-in-kubelet-plugin-watcher)\n    - [Enhancements in CSI Node Driver Registrar](#enhancements-in-csi-node-driver-registrar)\n    - [New Component: CSI Proxy](#new-component-csi-proxy)\n      - [CSI Proxy Named Pipes](#csi-proxy-named-pipes)\n      - [CSI Proxy Configuration](#csi-proxy-configuration)\n      - [CSI Proxy GRPC API](#csi-proxy-grpc-api)\n      - [CSI Proxy GRPC API Graduation and Deprecation Policy](#csi-proxy-grpc-api-graduation-and-deprecation-policy)\n      - [CSI Proxy Event Logs](#csi-proxy-event-logs)\n    - [Enhancements in Kubernetes/Utils/mounter](#enhancements-in-kubernetesutilsmounter)\n    - [Enhancements in CSI Node Plugins](#enhancements-in-csi-node-plugins)\n  - [Risks and Mitigations](#risks-and-mitigations)\n    - [Mitigation using PSP](#mitigation-using-psp)\n    - [Mitigation using a webhook](#mitigation-using-a-webhook)\n    - [Comparison of risks with CSI Node Plugins in Linux](#comparison-of-risks-with-csi-node-plugins-in-linux)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n  - [Graduation Criteria](#graduation-criteria)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n  - [Version Skew Strategy](#version-skew-strategy)\n- [Implementation History](#implementation-history)\n- [Drawbacks](#drawbacks)\n- [Alternatives](#alternatives)\n  - [API Alternatives](#api-alternatives)\n  - [Deployment Alternatives](#deployment-alternatives)\n    - [Deploying CSI Node Plugins as binaries and deployed as processes running on the host:](#deploying-csi-node-plugins-as-binaries-and-deployed-as-processes-running-on-the-host)\n    - [Package CSI Node Plugins as containers and deployed as processes running on the host:](#package-csi-node-plugins-as-containers-and-deployed-as-processes-running-on-the-host)\n    - [Support for Privileged Operations and Bi-directional mount propagation in Windows containers:](#support-for-privileged-operations-and-bi-directional-mount-propagation-in-windows-containers)\n- [Infrastructure Needed](#infrastructure-needed)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n- [x] kubernetes/enhancements issue in release milestone, which links to KEP (this should be a link to the KEP location in kubernetes/enhancements, not the initial KEP PR)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [ ] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [ ] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n## Summary\n\nContainer Storage Interface ([CSI](https://github.com/container-storage-interface/spec/blob/master/spec.md)) is a modern GRPC based standard for implementing external storage plugins (maintained by storage vendors, cloud providers, etc.) for container orchestrators like Kubernetes. Persistent storage requirements of containerized workloads can be satisfied from a diverse array of storage systems by installing and configuring the CSI plugins supported by the desired storage system. This KEP covers the enhancements necessary in Kubernetes core and CSI related out-of-tree components (specific to Kubernetes) to support CSI plugins for Windows nodes in a Kubernetes cluster. With the enhancements proposed in this KEP, Kubernetes operators will be able to leverage modern CSI plugins to satisfy the persistent storage requirements of Windows workloads in Kubernetes.\n\n## Motivation\n\nSupport for containerized Windows workloads on Windows nodes in a Kubernetes cluster reached GA status in v1.14. For persistent storage requirements, Windows workloads today depend on: (1) Powershell based [FlexVolume](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-storage/flexvolume.md) [plugins](https://github.com/microsoft/K8s-Storage-Plugins/tree/master/flexvolume/windows) maintained by Microsoft that support mounting remote storage volumes over SMB and iSCSI protocols and (2) In-tree [plugins](https://kubernetes.io/docs/concepts/storage/volumes/#types-of-volumes) in Kubernetes core (kubernetes/kubernetes repository) for popular cloud environments that support formatting and mounting direct attached disks on Windows nodes.\n\nSupport for CSI in Kubernetes reached GA status in v1.13. CSI plugins provide several benefits to Linux workloads in Kubernetes today over plugins whose code lives in kubernetes/kubernetes as well as plugins that implement the Flexvolume plugin interface. Some of these benefits are:\n\n1. The GRPC based CSI interface allow CSI plugins to be distributed as containers and fully managed through standard Kubernetes constructs like StatefulSets and DaemonSets. This is a superior mechanism compared to the exec model used by the Flexvolume interface where the plugins are distributed as scripts or binaries that need to be installed on each node and maintained. \n\n2. CSI offers a rich set of volume management operations (although not at a GA state in Kubernetes yet): resizing of volumes, backup/restore of volumes using snapshots and cloning besides the basic volume life-cycle operations (GA since v1.13): provisioning of storage volumes, attaching/detaching volumes to a node and mounting/dismounting to/from a pod. \n\n3. CSI plugins are maintained and released independent of the Kubernetes core. This allows features and bug fixes in the CSI plugins to be delivered in a more flexible schedule relative to Kubernetes releases. Transitioning the code for existing in-tree plugins (especially those targeting specific cloud environments or vendor-specific storage systems) to external CSI plugins can also help reduce the volume of vendor-ed code that needs to be maintained in Kubernetes core. \n\nGiven the above context, the main motivations for this KEP are:\n\n1. Enable Windows nodes to support CSI plugins so they can surface the above mentioned benefits of CSI plugins to Windows workloads that have persistent storage requirements. CSI Node Plugins today need to execute several privileged operations like scanning for newly provisioned disks, creating partitions on the disks, formatting the partitions with the desired file system as well as resize the filesystem, staging the partitions at a unique path on the host and propagating the staging path to workload containers. However, Windows does not support privileged operations from inside a container today. This KEP describes a host OS proxy to execute privileged operations on the Windows host OS on behalf of a container. The host OS proxy enables: [a] ease of distribution of CSI Node Plugins as containers for both Windows and Linux, [b] execution of CSI Node Plugins on Windows hosts in a manner similar to Linux hosts - from inside a container and [c] management of the CSI Node Plugin containers through Kubernetes constructs like Pods and DaemonSets.\n\n2. The CSI migration initiative (planned to reach beta state in v1.16) aims to deprecate the code associated with several in-tree storage plugins and pave the path for the ultimate removal of that code from Kubernetes core in favor of CSI plugins that implement the same functionality. Windows workloads need to be aligned with the CSI migration effort and cannot depend on environment specific in-tree plugins to satisfy persistent storage needs.\n\n### Goals\n\n1. Support all CSI Node Plugin operations: NodeStageVolume/NodeUnstageVolume, NodePublishVolume/NodeUnPublishVolume, NodeExpandVolume, NodeGetVolumeStats, NodeGetCapabilities and NodeGetInfo on Windows nodes.\n\n2. Support CSI plugins associated with a variety of external storage scenarios: block storage surfaced through iSCSI as well as directly attached disks (e.g. in cloud environments) as well as remote volumes over SMB.\n\n3. Ability to distribute CSI Node Plugins targeting Windows nodes as containers that can be deployed using DaemonSets in a Kubernetes cluster comprising of Windows nodes.\n\n### Non-Goals\n\n1. Support CSI Controller Plugin operations from Windows nodes: This may be considered in the future but not an immediate priority. Note that this does not require support for privileged operations on a Windows node as required by CSI Node Plugins and thus orthogonal to this KEP around CSI Node Plugins for Windows. If all the worker nodes in the cluster are Windows nodes and Linux master nodes have scheduling disabled then CSI controller plugins cannot be scheduled for now.\n\n2. Support privileged operations from Windows containers beyond CSI Node Plugins: This KEP introduces a host based \"privileged proxy\" process that may be used for executing privileged operations on the host on behalf of a Windows container. While a similar mechanism may be used for other use cases like containerized CNI plugins (for executing HNS operations), we leave that for a separate KEP. Scoping down the set of actions allowed by the API exposed by by the privileged proxy process to a minimal set simplifies multiple versions of the API as well as reduces the scope for abuse.\n\n3. Support for CSI plugins associated with external storage that requires a special file or block protocol kernel mode driver installed and configured on Windows hosts: e.g. FCoE (Fibre Channel over Ethernet), NFS volumes on Windows and Dokany based filesystems (https://github.com/dokan-dev/dokany) like SSHFS, etc.\n\n## Proposal\n\nIn this KEP, we propose a set of enhancements in pre-existing components to support CSI Node Plugins on Windows nodes.\n\nThe following enhancements are necessary in existing Kuberentes community managed code:\n1. Ability to handle Windows file paths in the Kubelet plugin watcher for domain sockets on Windows nodes.\n2. Refactor code in the CSI Node Driver Registrar so that it can be compiled for Windows.\n3. Build official CSI Node Driver Registrar container images based on Windows base images and publish them in official CSI community container registry.\n\nThe following enhancements are necessary in CSI plugins maintained by CSI plugin authors:\n1. Refactor code in existing CSI Node Plugins to support Windows. All privileged operations will need to be driven through an API exposed by a \"privileged proxy\" binary described below. Details around this will be documented in a plugin developer guide.\n2. Build CSI Node Plugin container images based on Windows base images.\n3. Create DaemonSet YAMLs referring to official CSI Node Driver Registrar container images and CSI Node Plugin container images targeting Windows.\n\nBesides the above enhancements, a new \"privileged proxy\" binary, named csi-proxy.exe is a key aspect of this KEP. csi-proxy.exe will run as a native Windows process on the Windows nodes configured as a Windows Service. csi-proxy.exe will expose an API (through GRPC over a named pipe) for executing privileged storage related operations on Windows hosts on behalf of Windows containers like CSI Node Plugins.\n\n### User Stories\n\nWith the KEP implemented, administrators should be able to deploy CSI Node Plugins that support staging, publishing and other storage management operations on Windows nodes. Operators will be able to schedule Windows workloads that consume persistent storage surfaced by CSI plugins on Windows nodes.\n\n#### Deploy CSI Node Plugin DaemonSet targeting Windows nodes \n\nAn administrator should be able to deploy a CSI Node Plugin along with the CSI Node Driver Registrar container targeting Windows nodes with a DaemonSet YAML like the following:\n\n```\nkind: DaemonSet\napiVersion: apps/v1\nmetadata:\n  name: csi-gce-pd-node-win\nspec:\n  selector:\n    matchLabels:\n      app: gcp-compute-persistent-disk-csi-driver-win\n  template:\n    metadata:\n      labels:\n        app: gcp-compute-persistent-disk-csi-driver-win\n    spec:\n      serviceAccountName: csi-node-sa\n      tolerations:\n      - key: \"node.kubernetes.io/os\"\n        operator: \"Equal\"\n        value: \"win1809\"\n        effect: \"NoSchedule\"\n      nodeSelector:\n        kubernetes.io/os: windows\n      containers:\n        - name: csi-driver-registrar\n          image: gke.gcr.io/csi-node-driver-registrar:win-v1 \n          args:\n            - \"--v=5\"\n            - \"--csi-address=unix://C:\\\\csi\\\\csi.sock\"\n            - \"--kubelet-registration-path=C:\\\\var\\\\lib\\\\kubelet\\\\plugins\\\\pd.csi.storage.gke.io\\\\csi.sock\"\n          env:\n            - name: KUBE_NODE_NAME\n              valueFrom:\n                fieldRef:\n                  fieldPath: spec.nodeName\n          volumeMounts:\n            - name: plugin-dir\n              mountPath: C:\\csi\n            - name: registration-dir\n              mountPath: C:\\registration\n        - name: gce-pd-driver\n          image: gke.gcr.io/gcp-compute-persistent-disk-csi-driver:win-v1 \n          args:\n            - \"--v=5\"\n            - \"--endpoint=unix:/csi/csi.sock\"\n          volumeMounts:\n            - name: kubelet-dir\n              mountPath: C:\\var\\lib\\kubelet\n            - name: plugin-dir\n              mountPath: C:\\csi\n            - name: csi-proxy-pipe\n              mountPath: \\\\.\\pipe\\csi-proxy-v1alpha1\n      volumes:\n        - name: csi-proxy-pipe\n          hostPath: \n            path: \\\\.\\pipe\\csi-proxy-v1alpha1\n            type: \"\"\n        - name: registration-dir\n          hostPath:\n            path: C:\\var\\lib\\kubelet\\plugins_registry\\\n            type: Directory\n        - name: kubelet-dir\n          hostPath:\n            path: C:\\var\\lib\\kubelet\\\n            type: Directory\n        - name: plugin-dir\n          hostPath:\n            path: C:\\var\\lib\\kubelet\\plugins\\pd.csi.storage.gke.io\\\n            type: DirectoryOrCreate\n```\n\nNote that references to GCE PD CSI Plugin is used as an example above based on a prototype port of GCE PD CSI plugin with the enhancements in this KEP. Controller pods for the CSI plugin can be deployed on Linux nodes in the cluster in the same manner as it is done today.\n\n#### Deploy Windows workloads that consume persistent storage managed by a CSI plugin\n\nAn operator should be able to deploy a Windows workload like SQL Server that consumes dynamically provisioned Persistent Volumes managed by a CSI plugin using:\n\nA storage class like:\n```\napiVersion: storage.k8s.io/v1\nkind: StorageClass\nmetadata:\n  name: csi-gce-pd\nprovisioner: pd.csi.storage.gke.io\nparameters:\n  type: pd-standard\n```\n\nwith a PVC like:\n```\napiVersion: v1\nmetadata:\n  name: sqlpvc\nspec:\n  accessModes:\n    - ReadWriteOnce\n  storageClassName: csi-gce-pd\n  resources:\n    requests:\n      storage: 100Gi\n```\n\nand a Pod like:\n```\napiVersion: v1\nkind: Pod\nmetadata:\n  name: sqlserver1\nspec:\n  tolerations:\n  - key: \"node.kubernetes.io/os\"\n    operator: \"Equal\"\n    value: \"win1809\"\n    effect: \"NoSchedule\"\n  nodeSelector:\n    beta.kubernetes.io/os: windows\n  containers:\n  - name: sqlpod\n    image: ddebroy/sqlsrv:latest\n    volumeMounts:\n    - name: csi-sql-vol\n      mountPath: C:\\Data\n    env:\n    - name: ACCEPT_EULA\n      value: \"y\"\n    - name: sa_password\n      value: \"*****\"\n  volumes:\n    - name: csi-sql-vol\n      persistentVolumeClaim:\n        claimName: sqlpvc\n```\n\n### Implementation Details\n\nCSI Node Plugins listen on domain sockets and respond to CSI API requests sent over GRPC from a container orchestrator like Kubernetes. They are responsible all storage management operations scoped around a specific node that are typically necessary after a CSI Controller Plugin has finished provisioning a Persistent Volume and attached it to the node. In Kubernetes, the CSI Node API calls are invoked by the CSI In-tree Plugin in the kubelet as well as the CSI Node Driver Registrar. The CSI Node Driver Registrar interacts with the Kubelet Plugin Watcher and it is maintained by the Kubernetes CSI community as a side-car container for deployment in CSI Node Plugin pods.\n\n![Kubernetes CSI Components](https://raw.githubusercontent.com/kubernetes/community/master/contributors/design-proposals/storage/container-storage-interface_diagram1.png?raw=true \"Kubernetes CSI Components\")\n\nSupport for Unix Domain Sockets has been introduced in Windows Server 2019 and works across containers as well as host and container as long as the processes running in containers are listening on the socket. If a process from within a container wishes to connect to a domain socket that a process on the host OS is listening on, Windows returns a permission error. This scenario however does not arise in the context of interactions between Kubelet, CSI Node Driver Registrar and CSI Node Plugin as these involve a process in a container listening on a domain socket (CSI Node Driver Registrar or CSI Node Plugin) that a process on the host (Kubelet) connects to.\n\nGolang supports domain socket operations for Windows since go version 1.12. It was found that in Windows, `os.ModeSocket` is not set on the `os.FileMode` associated with domain socket files in Windows. This issue is tracked [here](https://github.com/golang/go/issues/33357). Therefore determining whether a file is a domain socket file using `os.ModeSocket` does not work on Windows right now. We can potentially work around this by sending down a FSCTL to the file and evaluating the Windows reparse points to determine if the file is backed by a domain socket.\n\nBased on the above, we can conclude that some of the fundamental support in the OS and compiler with regards to domain sockets in the context of CSI plugin discovery and a channel for API invocation is present in a stable state in Windows Server 2019 today. Although there are some observed limitations with respect to domain sockets in Windows Server 2019, they are not major blockers in the context of CSI Node Plugins. In the section below, we call out the components in the context of CSI Node Plugins in Kubernetes that will need to be enhanced to properly account for Windows paths and make use of domain sockets in Windows in a manner very similar to Linux.\n\nCSI Node Plugins need to execute certain privileged operations at the host level as well as propagate mount points in response to the CSI API calls. Such operations involve: scanning disk identities to map the node OS's view of a disk device to a CSI volume provisioned and attached by CSI controller plugins, partitioning a disk and formatting it when necessary, bind-mounting volumes from the host to the container workload, resizing of the file system as part of a volume resize, etc. These operations cannot be invoked from a container in Windows today. As a result containerized CSI Node Plugins in Windows require some mechanism to perform these privileged operations on their behalf on the Windows host OS. csi-proxy.exe, described below serves that role by performing the storage related privileged operations on behalf of containers. Alternative approaches to csi-proxy.exe (for example, deploying the CSI plugin as regular binaries on the host without any containers) are described further below in the Alternatives section.\n\n#### Enhancements in Kubelet Plugin Watcher\n\nRegistration of CSI Node Plugins on a Kubernetes node is handled by the Kubelet plugin watcher using the fsnotify package. This component needs to convert paths detected by fsnotify to Windows paths in handleCreateEvent() and handleDeleteEvent() before the paths are passed to AddOrUpdatePlugin() RemovePlugin() routines in desiredStateOfTheWorld. A new utility function, NormalizeWindowsPath(), will be added in utils to handle this.\n\nGiven `os.ModeSocket` is not set on a socket file's `os.FileMode` in Windows (due to golang [issue](https://github.com/golang/go/issues/33357)), a specific check for `os.ModeSocket` in handleCreateEvent() will need to be relaxed for Windows until the golang issue is addressed.\n\n#### Enhancements in CSI Node Driver Registrar\n\nThe code for the CSI Node Driver Registrar needs to be refactored a bit so that it cleanly compiles when GOOS=windows is set. This mainly requires removal of dependencies on golang.org/x/sys/unix from nodeRegister() when building on Windows nodes.\n\nOnce compiled for Windows, container images based on Window Base images (like NanoServer) needs to be published and maintained.\n\n#### New Component: CSI Proxy \n\nA \"privileged proxy\" binary, csi-proxy.exe, will need to be developed and maintained by the Kubernetes CSI community to allow containerized CSI Node Plugins to perform privileged operations at the Windows host OS layer. Kubernetes administrators will need to install and maintain csi-proxy.exe on all Windows nodes in a manner similar to dockerd.exe today or containerd.exe in the future. A Windows node will typically be expected to be configured to run only one instance of csi-proxy.exe as a Windows Service that can be used by all CSI Node Plugins.\n\n##### CSI Proxy Named Pipes\nA CSI Node Plugin will interact with csi-proxy.exe using named pipe: `\\\\.\\pipe\\csi-proxy-v[N]` (exposed by csi-proxy.exe). The `v[N]` suffix in the pipe name corresponds to the version of the CSIProxyService (described below) that is required by the CSI plugin. Specific example of named pipes corresponding to versions include: `\\\\.\\pipe\\csi-proxy-v1`, `\\\\.\\pipe\\csi-proxy-v2alpha1`, `\\\\.\\pipe\\csi-proxy-v3beta1`, etc. The pipe will need to be mounted into the Node Plugin container from the host using the pod's volume mount specifications. Note that domain sockets cannot be used in this scenario since Windows blocks a containerized process from interacting with a host process that is listening on a domain socket. If such support is enabled on Windows in the future, we may consider switching CSI Proxy to use domain sockets however Windows named pipes is a common mechanism used by containers today to interact with host processes like docker daemon.\n\n![Kubernetes CSI Node Components and Interactions](csi-proxy3.png?raw=true \"Kubernetes CSI Node Components and Interactions\")\n\nA GRPC based interface, CSIProxyService, will be used by CSI Node Plugins to invoke privileged operations on the host through csi-proxy.exe. CSIProxyService will be versioned and any release of csi-proxy.exe binary will strive to maintain backward compatibility across as many prior stable versions of the API as possible. This avoids having to run multiple versions of the csi-proxy.exe binary on the same Windows host if multiple CSI Node Plugins (that do not depend on APIs in Alpha or Beta versions) have been configured and the version of the csi-proxy API required by the plugins are different. For every version of the API supported, csi-proxy.exe will first probe for and then expose a `\\\\.\\pipe\\csi-proxy-v[N]` pipe where v[N] can be v1, v2alpha1, v3beta1, etc. If during the initial probe phase, csi-proxy.exe determines that another process is already listening on a `\\\\.\\pipe\\csi-proxy-v[N]` named pipe, it will not try to create and listen on that named pipe. This allows multiple versions of csi-proxy.exe to run side-by-side if absolutely necessary to support multiple CSI Node Plugins that require widely different versions of CSIProxyService that no single version of csi-proxy.exe can support.\n\n##### CSI Proxy Configuration\nThere will be two command line parameters that may be passed to csi-proxy.exe:\n1. kubelet-csi-plugins-path: String parameter pointing to the path used by Kubernetes CSI plugins on each node. Will default to: `C:\\var\\lib\\kubelet\\plugins\\kubernetes.io\\csi`. All requests for creation and deletion of paths through the CSIProxyService RPCs (detailed below) will need to be under this path.\n\n2. kubelet-pod-path: String parameter pointing to the path used by the kubelet to store pod specific information. This should map to the value returned by [getPodsDir](https://github.com/kubernetes/kubernetes/blob/e476a60ccbe25581f5a6a9401081dcee311a066e/pkg/kubelet/kubelet_getters.go#L48). By default it will be set to: `C:\\var\\lib\\kubelet\\pods` Parameters to `LinkPath` (detailed below) will need to be under this path.\n\n##### CSI Proxy GRPC API\nThe following are the main RPC calls that will comprise a v1alpha1 version of the CSIProxyService API. A preliminary structure for Request and Response associated with each RPC is described below. Note that the specific structures as well as restrictions on them are expected to evolve during Alpha phase and are expected to be in a final form at the end of Beta. As the API evolves, the section below will be kept up-to-date\n\n```\nservice CSIProxyService {\n\n    // PathExists checks if the given path exists in the host already\n    rpc PathExists(PathExistsRequest) returns (PathExistsResponse) {}\n\n    // Mkdir creates a directory at the requested absolute path in the host. Relative path is not supported.\n    rpc Mkdir(MkdirRequest) returns (MkdirResponse) {}\n\n    // Rmdir removes the directory at the requested absolute path in the host. Relative path is not supported.\n    // This may be used for unlinking a symlink created through LinkVolume\n    rpc Rmdir(RmdirRequest) returns (RmdirResponse) {}\n\n    // Rescan refreshes the host storage cache\n    rpc Rescan(RescanRequest) returns (RescanResponse) {}\n\n    // PartitionDisk initializes and partitions a disk device (if the disk has not\n    // been partitioned already) and returns the resulting volume object\n    rpc PartitionDisk(PartitionDiskRequest) returns (PartitionDiskResponse) {}\n\n    // FormatVolume formats a volume with the provided file system.\n    // The resulting volume is mounted at the requested global staging path.\n    rpc FormatVolume(FormatVolumeRequest) returns (FormatVolumeResponse) {}\n\n    // MountSMBShare mounts a remote share over SMB on the host at the requested global staging path.\n    rpc MountSMBShare(MountSMBShareRequest) returns (MountSMBShareResponse) {}\n\n    // MountISCSILun mounts a remote LUN over iSCSI and returns the OS disk device number.\n    rpc MountISCSILun(MountISCSILunRequest) returns (MountISCSILunResponse) {}\n\n    // LinkPath invokes mklink on the global staging path of a volume linking it to a path within a container\n    rpc LinkPath(LinkPathRequest) returns (LinkPathResponse) {}\n\n    // ListDiskLocations returns locations \u003cAdapter, Bus, Target, LUN ID\u003e of all disk devices enumerated by Windows\n    rpc ListDiskLocations(ListDiskLocationsRequest) returns (ListDiskLocationsResponse) {}\n\n    // ListDiskIDs returns all IDs (from IOCTL_STORAGE_QUERY_PROPERTY) of all disk devices enumerated by Windows\n    rpc ListDiskIDs(GetDiskIDsRequest) returns (ListDiskIDsResponse) {}\n\n    // ListDiskVolumeMappings returns a map of all disk devices and volumes GUIDs\n    rpc ListDiskVolumeMappings(ListDiskVolumeMappingsRequest) returns (ListDiskVolumeMappingsResponse) {}\n\n    // ResizeVolume performs resizing of the partition and file system for a block based volume\n    rpc ResizeVolume(ResizeVolumeRequest) returns (ResizeVolumeResponse) {}\n\n    // DismountVolume gracefully dismounts a volume\n    rpc DismountVolume(DismountVolumeRequest) return (DismountVolumeResponse) {}\n}\n\nmessage PathExistsRequest {\n    // The path to check in the host filesystem.\n    string path = 1;\n}\n\nmessage PathExistsResponse {\n    // Whether path already exists in host or not\n    bool exists = 1;\n}\n\n// Context of the paths used for path prefix validation\nenum PathContext {\n    // plugin maps to the configured kubelet-csi-plugins-path path prefix\n    PLUGIN = 0;\n    // container maps to the configured kubelet-pod-path path prefix\n    CONTAINER = 1;\n}\n\nmessage MkdirRequest {\n    // The path to create in the host filesystem.\n    // All special characters allowed by Windows in path names will be allowed\n    // except for restrictions noted below. For details, please check:\n    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file\n    // Non-existent parent directories in the path will NOT be created.\n    // Directories will be created with Read and Write privileges of the Windows\n    // User account under which csi-proxy is started (typically LocalSystem).\n    //\n    // Restrictions:\n    // Needs to be an absolute path under kubelet-csi-plugins-path\n    // or kubelet-pod-path based on context\n    // Cannot exist already in host\n    // Path needs to be specified with drive letter prefix: \"X:\\\".\n    // UNC paths of the form \"\\\\server\\share\\path\\file\" are not allowed.\n    // All directory separators need to be backslash character: \"\\\".\n    // Characters: .. / : | ? * in the path are not allowed.\n    // Maximum path length will be capped to 260 characters (MAX_PATH).\n    string path = 1;\n\n    // Context of the path creation used for path prefix validation\n    PathContext context = 2;\n}\n\nmessage MkdirResponse {\n    // Windows error code\n    // Success is represented as 0\n    int32 error_code = 1;\n}\n\nmessage RmdirRequest {\n    // The path to remove in the host filesystem\n    // All special characters allowed by Windows in path names will be allowed\n    // except for restrictions noted below. For details, please check:\n    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file\n    //\n    // Restrictions:\n    // Needs to be an absolute path under kubelet-csi-plugins-path\n    // or kubelet-pod-path based on context\n    // Path needs to be specified with drive letter prefix: \"X:\\\".\n    // UNC paths of the form \"\\\\server\\share\\path\\file\" are not allowed.\n    // All directory separators need to be backslash character: \"\\\".\n    // Characters: .. / : | ? * in the path are not allowed.\n    // Path cannot be a file of type symlink\n    // Path needs to be a directory that is empty\n    // Maximum path length will be capped to 260 characters (MAX_PATH).\n    string path = 1;\n\n    // Context of the path creation used for path prefix validation\n    PathContext context = 2;\n}\n\nmessage RmdirResponse {\n    // Windows error code\n    // Success is represented as 0\n    int32 error_code = 1;\n}\n\nmessage RescanRequest {\n    // Intentionally empty.\n}\n\nmessage RescanResponse {\n    // Windows error code\n    // Success is represented as 0\n    int32 error_code = 1;\n}\n\nmessage PartitionDiskRequest {\n    // The Windows disk device to partition and the paritioning mode: MBR/GPT.\n    // The whole disk will be partitioned\n    //\n    // Restrictions:\n    // Disk device needs to follow Win32 format for device names: \\\\.\\PhysicalDriveX\n    // The prefix has to be: \\\\.\\PhysicalDrive and the suffix is an integer in range:\n    // 0 to maximum number drives allowed by Windows OS.\n    string disk_device = 1;\n\n    // Disk partition type\n    enum ParitionType {\n        MBR = 0;\n        GPT = 1;\n    }\n    PartitionType type = 2;\n}\n\nmessage PartitionDiskResponse {\n    // Volume device resulting from the partition\n    // Volume device will follow Win32 namespaced GUID format for volumes: \\\\?\\Volume\\{GUID}\n    // The prefix has to be: \\\\?\\Volume\\ and the suffix to be a GUID enclosed with {}\n    string volume_device = 1;\n\n    // Windows error code\n    // Success is represented as 0\n    int32 error_code = 2;\n}\n\nmessage FormatVolumeRequest {\n    // The Windows volume device to format\n    // Typically Volume Device returned by PartitionDiskResponse\n    //\n    // Restrictions:\n    // Volume device needs to follow Win32 namespaced GUID format for volumes: \\\\?\\Volume\\{GUID}\n    // The prefix has to be: \\\\?\\Volume\\ and the suffix to be a GUID enclosed with {}\n    string volume_device = 1;\n\n    // FileSystem type\n    enum FileSystemType {\n        NTFS = 0;\n        FAT = 1;\n    }\n    FileSystemType type = 2;\n}\n\nmessage FormatVolumeResponse {\n    // Windows error code\n    // Success is represented as 0\n    int32 error_code = 1;\n}\n\nmessage MountSMBShareRequest {\n    // A remote SMB share to mount\n    // All unicode characters allowed in SMB server name specifications are\n    // permitted except for restrictions below\n    //\n    // Restrictions:\n    // SMB share specified in the format: \\\\server-name\\sharename, \\\\server.fqdn\\sharename or \\\\a.b.c.d\\sharename\n    // If not an IP address, share name has to be a valid DNS name.\n    // UNC specifications to local paths or prefix: \\\\?\\ is not allowed.\n    // Characters: + [ ] \" / : ; | \u003c \u003e , ? * = $ are not allowed.\n    string remote_share = 1;\n\n    // Local path in the host to stage the SMB share.\n    // All special characters allowed by Windows in path names will be allowed\n    // except for restrictions noted below. For details, please check:\n    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file\n    //\n    // Restrictions:\n    // Needs to be an absolute path under kubelet-csi-plugins-path.\n    // Needs to exist already in host\n    // Path needs to be specified with drive letter prefix: \"X:\\\".\n    // UNC paths of the form \"\\\\server\\share\\path\\file\" are not allowed.\n    // All directory separators need to be backslash character: \"\\\".\n    // Characters: .. / : | ? * in the path are not allowed.\n    // Maximum path length will be capped to 260 characters (MAX_PATH).\n    string host_path = 2;\n\n    // Mount the share read-only\n    bool readonly = 3;\n\n    // Username credential associated with the share\n    string username = 4;\n\n    // Password credential associated with the share\n    string password = 5;\n}\n\nmessage MountSMBShareResponse {\n    // Windows error code\n    // Success is represented as 0\n    int32 error_code = 1;\n}\n\nmessage MountISCSILunRequest {\n    // IQN address\n    // follows IQN format: iqn.yyyy-mm.naming-authority:unique name\n    string node_iqn = 1;\n\n    // Authentication Type\n    enum AuthType {\n        None = 0;\n        OneWay = 1;\n        Mutual = 2;\n    }\n    AuthType auth_type = 2;\n\n    // Discovery CHAP username\n    string discovery_chap_username = 3;\n\n    // Discovery CHAP secret\n    string discovery_chap_secret = 4;\n\n    // Session CHAP username\n    string session_chap_username = 5;\n\n    // Session CHAP secret\n    string session_chap_secret = 6;\n\n    // TargetPortal address\n    string target_portal_address = 7;\n\n    // TargetPortal port\n    string target_portal_port = 8;\n\n    // Readonly mount\n    bool readonly = 9;\n}\n\nmessage MountISCSILunResponse {\n    // Windows error code\n    // Success is represented as 0\n    int32 error_code = 1;\n}\n\nmessage LinkPathRequest {\n    // Source of MkLink call to Windows\n    // All special characters allowed by Windows in path names will be allowed\n    // except for restrictions noted below. For details, please check:\n    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file\n    //\n    // Restrictions:\n    // Needs to be an absolute path under kubelet-pod-path\n    // Needs to exist already in host\n    // Path needs to be specified with drive letter prefix: \"X:\\\".\n    // UNC paths of the form \"\\\\server\\share\\path\\file\" are not allowed.\n    // All directory separators need to be backslash character: \"\\\".\n    // Characters: .. / : | ? * in the path are not allowed.\n    // Maximum path length will be capped to 260 characters (MAX_PATH).\n    string source_path = 1;\n\n    // Target of MkLink call to Windows\n    // All special characters allowed by Windows in path names will be allowed\n    // except for restrictions noted below. For details, please check:\n    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file\n    //\n    // Restrictions:\n    // Needs to be an absolute path under kubelet-csi-plugins-path.\n    // Needs to exist already in host\n    // Path needs to be specified with drive letter prefix: \"X:\\\".\n    // UNC paths of the form \"\\\\server\\share\\path\\file\" are not allowed.\n    // All directory separators need to be backslash character: \"\\\".\n    // Characters: .. / : | ? * in the path are not allowed.\n    // Maximum path length will be capped to 260 characters (MAX_PATH).\n    string target_path = 2;\n}\n\nmessage LinkPathResponse {\n    // Windows error code\n    // Success is represented as 0\n    int32 error_code = 1;\n}\n\nmessage ListDiskLocationsRequest {\n    // Intentionally empty\n}\n\nmessage ListDiskLocationsResponse {\n    // Map of disk device objects and \u003cadapter, bus, target, lun ID\u003e associated with each disk device\n    map \u003cstring, DiskLocation\u003e disk_locations = 1;\n\n    // Windows error code\n    // Success is represented as 0\n    int32 error_code = 2;\n}\n\nmessage DiskLocation {\n    string Adapter = 0;\n    string Bus = 1;\n    string Target = 2;\n    string LUNID = 3;\n}\n\nmessage ListDiskIDsRequest {\n    // Intentionally empty\n}\n\nmessage ListDiskIDsResponse {\n    // Map of disk device objects and IDs associated with each disk device\n    map \u003cstring, DiskIDs\u003e disk_id = 1;\n\n    // Windows error code\n    // Success is represented as 0\n    int32 error_code = 2;\n}\n\nmessage DiskIDs {\n    // list of Disk IDs of ascii characters associated with disk device\n    repeated strings IDs = 1;\n}\n\nmessage ListDiskVolumeMappingsRequest {\n    // Intentionally empty\n}\n\nmessage ListDiskVolumeMappingsResponse {\n    // Map of disk devices and volume objects of the form \\\\?\\volume\\{GUID} on the disk\n    map \u003cstring, string\u003e disk_volume_pair = 1;\n\n    // Windows error code\n    // Success is represented as 0\n    int32 error_code = 2;\n}\n\nmessage ResizeVolumeRequest {\n    // The Win32 volume device to resize\n    //\n    // Restrictions:\n    // Volume device needs to follow Win32 namespaced GUID format for volumes: \\\\?\\Volume\\{GUID}\n    // The prefix has to be: \\\\?\\Volume\\ and the suffix to be a GUID enclosed with {}\n    string volume_device = 1;\n\n    // New size to resize FS to\n    int64 new_size = 2;\n}\n\nmessage ResizeVolumeResponse {\n    // Windows error code\n    // Success is represented as 0\n    int32 error_code = 1;\n}\n\nmessage DismountVolumeRequest {\n    // The Win32 volume device to dismount gracefully\n    //\n    // Restrictions:\n    // Volume device needs to follow Win32 namespaced GUID format for volumes: \\\\?\\Volume\\{GUID}\n    // The prefix has to be: \\\\?\\Volume\\ and the suffix to be a GUID enclosed with {}\n    string volume_device = 1;\n}\n\nmessage DismountVolumeResponse {\n    // Windows error code\n    // Success is represented as 0\n    int32 error_code = 1;\n}\n\n```\n\n##### CSI Proxy GRPC API Graduation and Deprecation Policy\n\nIn accordance with standard Kubernetes conventions, the above API will be introduced as v1alpha1 and graduate to v1beta1 and v1 as the feature graduates. Beyond a vN release in the future, new RPCs and enhancements to parameters will be introduced through vN+1alpha1 and graduate to vN+1beta1 and vN+1 stable versions as the new APIs mature.\n\nMembers of CSIProxyService API may be deprecated and then removed from csi-proxy.exe in a manner similar to Kubernetes deprecation (policy)[https://kubernetes.io/docs/reference/using-api/deprecation-policy/] although maintainers will make an effort to ensure such deprecation is as rare as possible. After their announced deprecation, a member of CSIProxyService API must be supported:\n1. 12 months or 3 releases (whichever is longer) if the API member is part of a Stable/vN version.\n2. 9 months or 3 releases (whichever is longer) if the API member is part of a Beta/vNbeta1 version.\n3. 0 releases if the API member is part of an Alpha/vNalpha1 version.\n\nTo continue running CSI Node Plugins that depend on an old version of csi-proxy.exe, vN, some of whose members have been removed, Kubernetes administrators will be required to run the latest version of the csi-proxy.exe (that will be used by CSI Node Plugins that use versions of CSIProxyService more recent than vN) along with an old version of csi-proxy.exe that does support vN.\n\nIntroduction of new RPCs or enhancements to parameters is expected to be inspired by new requirements from plugin authors as well as CSI functionality enhancement.\n\n##### CSI Proxy Event Logs\n\nFor all RPC invocations, csi-proxy.exe will log events to the Windows application event log. This will act as an audit trail that may be correlated with audit logs from Kubeneretes around operations involving PV creation to track potential unexpected invocation of CSIProxyService by a malicious pod or process.\n\n#### Enhancements in Kubernetes/Utils/mounter\n\nOnce the [PR](https://github.com/kubernetes/utils/pull/100/files) lands, a mounter/mount_windows_using_csi_proxy.go in Kubernetes/Utils/mounter package can be introduced. It will implement the mounter and hostutil interfaces against the CSIProxyService API.\n\n#### Enhancements in CSI Node Plugins\n\nCode for CSI Node Plugins need to be refactored to support CSI Node APIs in both Linux and Windows nodes. While the code targeting Linux nodes can assume privileged access to the host, the code targeting Windows nodes need to invoke the GRPC client API associated with the desired version of the CSIProxyService described above. CSI Node Plugins that will use the Kubernetes/Utils/mounter package introduced in this [PR](https://github.com/kubernetes/utils/pull/100/files) will require minimal platform specific code targeting Windows and Linux.\n\nOnce compiled for Windows, container images based on Window Base images (like NanoServer) needs to be published and maintained. Container images targeting Linux nodes will need to be based on the desired Linux distro base image.\n\nNew YAMLs for DaemonSets associated with CSI Node Plugins needs to be authored that will (1) target Windows nodes and (2) use Windows paths for domain socket related paths as illustrated in the User Story section above.\n\n### Risks and Mitigations\n\nAny pod on a Windows node can be configured to mount `\\\\.\\pipe\\csi-proxy-v[N]` and perform privileged operations. Thus csi-proxy presents a potential security risk. To mitigate the risk, some options are described below:\n\n#### Mitigation using PSP\nAdministrators can enable Pod Security Policy (PSP) in their cluster and configure PSPs to:\n1. Disallow host path mounts as part of a default cluster-wide PSP. This will affect all pods in the cluster across Linux and Windows that mount any host paths.\n2. Allow host path mounts with pathPrefix = `\\\\.\\pipe\\csi-proxy`. Restrict usage of this PSP to only SAs associated with the DaemonSets of CSI Node Plugins.\nSupport will need to be implemented in AllowsHostVolumePath to handle Windows pipe paths.\n\n#### Mitigation using a webhook\nAn admission webhook can be implemented and deployed in clusters with Windows nodes that will reject all containers that mount paths with prefix `\\\\.\\pipe\\csi-proxy` as a hostPath volume but does not have privileged flag set in the pod's securityContext specification. This allows the privileged setting to be used for Windows pods as an indication the container will perform privileged operations. Other cluster-wide policies (e.g. PSP) that act on the privileged setting in a container's securityContext can enforce the same for CSI Node plugins targeting Windows nodes. Note that this does not in any way change how the privileged setting is used today for Linux nodes. If in the future, full privileged container support is introduced in Windows (as in Linux today), functionality of existing CSI Node Plugin DaemonSets (targeting Windows) with the privileged flag set should not get negatively impacted as they will be launched as privileged containers.\n\n#### Comparison of risks with CSI Node Plugins in Linux\nIn Linux nodes, CSI Node Plugins typically surface a domain socket used by the kubelet to invoke CSI API calls on the node plugin. This socket may also be mounted by a malicious pod to invoke CSI API calls that may be potentially destructive - for example a Node Stage API call that leads to a volume being formatted. The malicious pod will have to correctly guess the parameters for the CSI API calls for a particular node plugin in order to influence it's behavior as well as circumvent validation checks (or exploit logical vulnerabilities/bugs) in the plugin's code. In case of Windows, a malicious pod can perform destructive actions using the csi-proxy pipe with far less barriers in the absence of the above mitigations. However the overall attack surface of using a mounted domain socket in Linux or a named pipe in Windows from a malicious pod is similar.\n\n## Design Details\n\n### Test Plan\n\nUnit tests will be added to verify Windows related enhancements in existing Kubernetes components mentioned above.\n\nAll E2E storage tests covering CSI plugins will be ported to Windows workloads and successfully executed with above enhancements in place along with csi-proxy.exe.\n\n### Graduation Criteria\n\n#### Alpha -\u003e Beta Graduation\n\n- csi-proxy.exe supports v1beta1 version of the CSIProxyService API.\n- end-2-end tests in place with a CSI plugin that can support Windows containers and pass all existing CSI plugin test scenarios.\n\n#### Beta -\u003e GA Graduation\n\n- In-tree storage plugins that implements support for Windows (AWS EBS, GCE PD, Azure File and Azure Disk as of today) can use csi-proxy.exe along with other enhancements listed above to successfully deploy CSI plugins on Windows nodes.\n- csi-proxy.exe supports v1 stable version of the CSIProxyService API.\n- Successful usage of csi-proxy.exe with support for v1 version of CSIProxyService API in Windows nodes by at least two storage vendors.\n\n### Upgrade / Downgrade Strategy\n\nIn order to install a CSI Node Plugin or upgrade to a version of a CSI Node Plugin that uses an updated version of the CSIProxyService API not supported by the currently deployed version of csi-proxy.exe in the cluster, csi-proxy.exe will need to be upgraded first on all nodes of the cluster before deploying or upgrading the CSI Node Plugin. In case there is a very old CSI Node Plugin in the cluster that relies on a version of CSIProxyService API that is no longer supported by the new version of csi-proxy.exe, the previously installed version of csi-proxy.exe should not be uninstalled from the nodes. Such scenarios are expected to be an exception.\n\nDifferent nodes in the cluster may be configured with different versions of csi-proxy.exe as part of a rolling upgrade of csi-proxy.exe. In such a scenario, it is recommended that csi-proxy.exe upgrade is completed first across all nodes. Once that is complete, the CSI Node Plugins that can take advantage of the new version of csi-proxy.exe may be deployed.\n\nDowngrading the version of csi-proxy.exe to one that is not supported by all installed versions the CSI Node Plugins in the cluster will lead to loss of access to data. Further, if a cluster is downgraded from a version of Kubernetes where the plugin watcher supports Windows nodes to one that does not, existing Windows workloads that were using CSI plugins to access storage will no longer have access to the data. This loss of functionality cannot be handled in an elegant fashion.\n\n### Version Skew Strategy\n\nBeyond the points in the above section (Upgrade/Downgrade strategy), there are no Kubernetes version skew considerations in the context of this KEP.\n\n## Implementation History\n\n07/16/2019: Initial KEP drafted\n\n07/20/2019: Feedback from initial KEP review addressed.\n\n## Drawbacks\n\nThe main drawback associated with the approach leveraging csi-proxy.exe is that the life cycle of that binary as well as logs will need to be managed out-of-band from Kubernetes. However, cluster administrators need to maintain and manage life cycle and logs of other core binaries like kubeproxy.exe, kubelet.exe, dockerd.exe and containerd.exe (in the future). Therefore csi-proxy.exe will be one additional binary that will need to be treated in a similar way.\n\nThe API versioning scheme described above, will try to maintain backward compatibility as much as possible. This requires the scope of csi-proxy.exe to be limited to a very scoped down fundamental set of operations. Maintainers therefore will need to be very cautious when accepting suggestions for new APIs and enhancements. This may slow progress at times.\n\nThere may ultimately be certain operations that csi-proxy.exe cannot support in an elegant fashion and require the plugin author targeting Windows nodes to seek one of the alternatives described below. There may also be a need to support volumes that do not use standard block or file protocols. In such scenarios, an extra payload (in the form of a binary, kernel driver and service) may need to be dropped on the host and maintained out-of-band from Kubernetes. This KEP and maintainers should ensure such instances are as limited as possible.\n\n## Alternatives\n\nThere are alternative approaches to the CSIProxyService API as well as the overall csi-proxy mechanism described in this KEP. These alternatives are enumerated below.\n\n### API Alternatives\n\nThe CSIProxyService API will be a defined set of operations that will need to expand over time as new CSI APIs are introduced that require new operations on every node as well as desire for richer operations by CSI plugin authors. Unfortunately this comes with a maintenance burden associated with tracking and graduating new RPCs across versions.\n\nAn alternative approach that simplifies the above involves exposing a single Exec interface in CSIProxyService that supports passing an arbitrary set of parameters to arbitrary executables and powershell cmdlets on the Windows host and collecting and returning the stdout and stderr back to the invoking containerized process. Since all the currently enumerated operations in the CSIProxyService API can be driven through the generic Exec interface, the set of desired privileged operations necessary becomes a decision for plugin authors rather than maintainers of csi-proxy. The main drawback of this highly flexible mechanism is that it drastically increases the potential for abuse in the host by 3rd party plugin authors. The ability to exploit bugs or vulnerabilities in individual plugins to take control of the host becomes much more trivial with a generic Exec RPC relative to exploiting other RPCs of the CSIProxyService API.\n\nDepending on the adoption of csi-proxy in the Alpha and Beta phases and the need for specialized privileged operations, we may consider adding a generic Exec interface in the future.\n\n### Deployment Alternatives\n\nThere are multiple alternatives to deploying CSI Node Plugins as containers along with csi-proxy.exe for privileged operations however each has it's own set of disadvantages.\n\n#### Deploying CSI Node Plugins as binaries and deployed as processes running on the host:\n\nWith this approach, lifecycle of multiple stand-alone binaries corresponding to different CSI node plugins will need to be managed. The standard CSI Node Driver Registrar which is distributed as a container will also need to be repackaged as binaries and distributed (as mixing side car containers with standalone binaries is not possible). Managing several binaries outside of Kubernetes may not scale well as diverse storage systems, each with their own CSI plugin, is integrated with the cluster. Since Kubernetes has no knowledge of these binaries, operators will not be able to use standard Kubernetes constructs to monitor and control the life-cycle of the binaries. Collection of logs from the stand-alone binaries will require tooling out-of-band from Kubernetes.\n\n#### Package CSI Node Plugins as containers and deployed as processes running on the host:\n\nWith this approach, the container run time is enhanced to be able to launch binaries directly on the host after pulling down a container image and extracting the binary from the image. While usual Kubernetes constructs may be used to launch pods with a special RuntimeClass that can handle launching of the binaries as processes on hosts, various enhancements will be necessary in the runtime to enable Kubernetes to fully monitor and control the whole life-cycle of the binaries post launch. Collection of logs from the plugins also become problematic and will require either out-of-band tooling at present or various enhancements in the runtime in the future.\n\n#### Support for Privileged Operations and Bi-directional mount propagation in Windows containers:\n\nAt some point in the future, a Windows LTS release may implement support for execution of privileged operations and bi-directional mount propagation from inside containers. At that point, the requirement of a proxy to handle privileged operations on behalf of containers will disappear. However, before such support is committed to and implemented in a Windows LTS release (which is not expected in at least a year), we need solutions as described in the KEP.\n\n## Infrastructure Needed\n\nThe code for csi-proxy as well as the GRPC API will be maintained in a dedicated repo: github.com/kubernetes-csi/windows-csi-proxy \n"
+  },
+  {
+    "id": "67f1fdacabf84296a10f132d9514caf5",
+    "title": "Windows RuntimeClass Support",
+    "authors": ["@patricklang"],
+    "owningSig": "sig-windows",
+    "participatingSigs": ["sig-node"],
+    "reviewers": ["@tallclair", "@derekwaynecarr", "@benmoss", "@ddebroy"],
+    "approvers": ["@dchen1107"],
+    "editor": "@patricklang",
+    "creationDate": "2019-10-08",
+    "lastUpdated": "2019-10-15",
+    "status": "implementable",
+    "seeAlso": ["/keps/sig-windows/20190424-windows-cri-containerd.md"],
+    "replaces": null,
+    "supersededBy": null,
+    "markdown": "\n# RuntimeClass Support for Windows\n\n## Table of Contents\n\n\u003c!-- toc --\u003e\n- [Release Signoff Checklist](#release-signoff-checklist)\n- [Summary](#summary)\n- [Motivation](#motivation)\n  - [Goals](#goals)\n  - [Non-Goals](#non-goals)\n- [Proposal](#proposal)\n  - [User Stories](#user-stories)\n  - [Implementation Details/Notes/Constraints](#implementation-detailsnotesconstraints)\n    - [Adding new label node.kubernetes.io/windows-build (done)](#adding-new-label-nodekubernetesiowindows-build-done)\n    - [Adding handler to CRI pull API](#adding-handler-to-cri-pull-api)\n  - [Risks and Mitigations](#risks-and-mitigations)\n    - [Adding new node label](#adding-new-node-label)\n    - [Adding runtime_handler to PullImageRequest](#adding-runtime_handler-to-pullimagerequest)\n- [Design Details](#design-details)\n  - [Test Plan](#test-plan)\n    - [E2E Testing with CRI-ContainerD and Kubernetes](#e2e-testing-with-cri-containerd-and-kubernetes)\n    - [Unit testing with CRITest](#unit-testing-with-critest)\n  - [Graduation Criteria](#graduation-criteria)\n      - [Removing a deprecated flag](#removing-a-deprecated-flag)\n  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)\n- [Implementation History](#implementation-history)\n- [Alternatives](#alternatives)\n  - [Support multiarch os/arch/version in CRI](#support-multiarch-osarchversion-in-cri)\n  - [Make the scheduler aware of Multi-arch images](#make-the-scheduler-aware-of-multi-arch-images)\n  - [Create a multi-arch Mutating admission controller](#create-a-multi-arch-mutating-admission-controller)\n- [Future Considerations](#future-considerations)\n  - [Pod Overhead](#pod-overhead)\n  - [RuntimeClass Parameters](#runtimeclass-parameters)\n- [Reference \u0026amp; Examples](#reference--examples)\n  - [Multi-arch container image overview](#multi-arch-container-image-overview)\n\u003c!-- /toc --\u003e\n\n## Release Signoff Checklist\n\n**ACTION REQUIRED:** In order to merge code into a release, there must be an issue in [kubernetes/enhancements] referencing this KEP and targeting a release milestone **before [Enhancement Freeze](https://github.com/kubernetes/sig-release/tree/master/releases)\nof the targeted release**.\n\nFor enhancements that make changes to code or processes/procedures in core Kubernetes i.e., [kubernetes/kubernetes], we require the following Release Signoff checklist to be completed.\n\nCheck these off as they are completed for the Release Team to track. These checklist items _must_ be updated for the enhancement to be released.\n\n- [x] [kubernetes/enhancements issue in release milestone](https://github.com/kubernetes/enhancements/issues/1301)\n- [ ] KEP approvers have set the KEP status to `implementable`\n- [x] Design details are appropriately documented\n- [ ] Test plan is in place, giving consideration to SIG Architecture and SIG Testing input\n- [x] Graduation criteria is in place\n- [ ] \"Implementation History\" section is up-to-date for milestone\n- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]\n- [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes\n\n**Note:** Any PRs to move a KEP to `implementable` or significant changes once it is marked `implementable` should be approved by each of the KEP approvers. If any of those approvers is no longer appropriate than changes to that list should be approved by the remaining approvers and/or the owning SIG (or SIG-arch for cross cutting KEPs).\n\n**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.\n\n[kubernetes.io]: https://kubernetes.io/\n[kubernetes/enhancements]: https://github.com/kubernetes/enhancements/issues\n[kubernetes/kubernetes]: https://github.com/kubernetes/kubernetes\n[kubernetes/website]: https://github.com/kubernetes/website\n\n## Summary\n\nRuntimeClass can be used to make it easier to schedule Pods onto appropriate nodes based on OS, OS Version, CPU Architecture and variant. These are all supported in container distribution \u0026 runtimes as part of [Multi-arch container images](#multi-arch-container-image-overview) today.\n\nWith Hyper-V available, Windows can run containers multiple Windows OS versions today, and Linux containers may be available in the future. This document proposes controlling those features through RuntimeClass as well, rather than adding new fields to the Pod API and changing Kubernetes scheduling behavior.\n\n## Motivation\n\nThere are a few related customer experience problems when running mixed (multiple-OS and/or multiple-CPU-arch) clusters today. Kubernetes scheduling is based solely on what's declared in the deployment as part of the Pod spec. A Pod could be scheduled to an incompatible node and fail to pull. This error causes a retry and backoff loop, and doesn't automatically fail over to another node.\n\nIn addition to the stuck pull failure case, there's also a question of user intent. Some containers are published for multiple OS's and/or architectures - this often referred to as multi-arch but the OCI distribution spec actually includes the OS, OS version, CPU architecture and variant. If a node can run multiple permutations of these, then there's not a deterministic way to know what the user intended to run. Running a container under CPU emulation may have performance penalties, or cost more in terms of leased compute time.\n\nToday with only a single supported Windows version (10.0.17763) \u0026 runtime (Docker), the problem is easily mitigated with NodeSelectors or Taints. This is documented today in [Guide for scheduling Windows containers in Kubernetes]. Moving these to RuntimeClasses simplifies the experience further, and can simplify deployment YAML for users in Kubernetes v1.16.\n\nNext, SIG-Windows is adding support for [Windows CRI-ContainerD], Windows nodes will be able to handle running multiple Windows OS versions using Hyper-V isolation. This technology could be used to run Linux containers as well, leading to more ambiguity. This KEP aims to resolve the ambiguity of pulling and running a container image and pod when multiple os/versions/architecture/variants may be supported on a single node.\n\n### Goals\n\n- Schedule a Windows Pod to a compatible node\n  - Allow matching Windows versions without Hyper-V isolation\n  - Allow opt-in on a per-Pod basis to run containers using existing Windows versions with backwards-compatibility provided by Hyper-V on a new Windows OS version node\n- Be able to schedule a specific image from a multi-arch manifest on a given node\n- Provide a simpler experience (fewer lines of YAML) than adding os and version nodeSelector and tolerations to each Pod\n\n### Non-Goals\n\n- Linux container support on Windows is not a requirement or test target for Kubernetes 1.17, but it's not specifically excluded.\n- Running newer Windows OS version containers (Windows Server version 1903) on an older OS version host (Windows Server version 1809). This is not supported today with or without Hyper-V isolation (see [Windows container version compatibility]).\n\n## Proposal\n\nFor Kubernetes 1.17, we're proposing three key things to improve the experience deploying Windows containers across multiple versions, and enable experimentation with Hyper-V while ContainerD support is in alpha/beta stages.\n\n1. Add `node.kubernetes.io/windows-build` label using Windows kubelet\n2. Add RuntimeHandler to the CRI pull API\n3. Use RuntimeHandler and separate ContainerD configurations to control runtime specific parameters such as overrides for OS/version/architecture and Hyper-V isolation.\n\nAs a fallback plan, steps 2/3 could use annotations instead during alpha, but this would make things more difficult for users trying out the new features as the Pod specs would change between versions. Comparable solutions based on ContainerD + Kata on Linux are moving away from annotations and to RuntimeClass, so we want to follow the same approach for ContainerD + Hyper-V.\n\n### User Stories\n\n#### Story 1 - Easy selection of Windows Server releases\n\nAs of Kubernetes 1.16, [RuntimeClass Scheduling] is in beta and can be used to simplify setting nodeSelector \u0026 tolerations. This makes it easier to steer workloads to a suitable Windows or Linux node using the existing labels. With the addition of a new `windows-build` label it can also distinguish between multiple Windows version in the same cluster. For more details on how and why build numbers are used, read [Adding new label node.kubernetes.io/windows-build](#adding-new-label-nodekubernetesiowindows-build) below.\n\n\u003e Note: There's an open PR [website#16697](https://github.com/kubernetes/website/pull/16697) to add `RuntimeClass` examples to existing documentation for Kubernetes 1.16.\n\n- Example of a RuntimeClass that would restrict a pod to Windows Server 2019 / 1809 (10.0.17763)\n\n    ```yaml\n    apiVersion: node.k8s.io/v1beta1\n    kind: RuntimeClass\n    metadata:\n      name: windows-1809\n    handler: 'docker'\n    scheduling:\n      nodeSelector:\n        kubernetes.io/os: 'windows'\n        kubernetes.io/arch: 'amd64'\n        node.kubernetes.io/windows-build: '10.0.17763'\n      tolerations:\n      - effect: NoSchedule\n        key: windows\n        operator: Equal\n        value: \"true\"\n    ```\n\n- Example of a RuntimeClass that would restrict a pod to Windows Server version 1903 (10.0.18362)\n\n    ```yaml\n    apiVersion: node.k8s.io/v1beta1\n    kind: RuntimeClass\n    metadata:\n      name: windows-1903\n    handler: 'docker'\n    scheduling:\n      nodeSelector:\n        kubernetes.io/os: 'windows'\n        kubernetes.io/arch: 'amd64'\n        node.kubernetes.io/windows-build: '10.0.18362'\n      tolerations:\n      - effect: NoSchedule\n        key: windows\n        operator: Equal\n        value: \"true\"\n    ```\n\n#### Story 2 - Forward compatibility with Hyper-V\n\nOnce a new version of Windows Server is deployed using [Windows CRI-ContainerD], Hyper-V can be enabled to provide backwards compatibility and run Windows containers based on the previous OS version. Cluster admins can pick between two different approaches to move applications forward. Currently Windows Server has only supported backwards compatibility with Hyper-V, for example running a 1809 container on 1903. Therefore the node OS version should be the same or ahead of the container OS version used (see: [Windows container version compatibility]).\n\n1. Create a new RuntimeClass that Pods can use to try out Hyper-V isolation on the new version of Windows.\n\n    ```yaml\n    apiVersion: node.k8s.io/v1beta1\n    kind: RuntimeClass\n    metadata:\n      name: windows-1809-hyperv\n    handler: 'containerd-hyperv-17763'\n    scheduling:\n      nodeSelector:\n        kubernetes.io/os: 'windows'\n        kubernetes.io/arch: 'amd64'\n        node.kubernetes.io/windows-build: '10.0.18362'\n      tolerations:\n      - effect: NoSchedule\n        key: windows\n        operator: Equal\n        value: \"true\"\n    ```\n\n1. Once sufficient testing is done, the RuntimeClass from Story 1 could be updated. This would cause new deployments to go to these nodes, without having to update them individually. The nodes running the previous Windows version could be drained and removed from the cluster. As the pods running on them are rescheduled, the new RuntimeClass will be applied.\n\n    ```yaml\n    apiVersion: node.k8s.io/v1beta1\n    kind: RuntimeClass\n    metadata:\n      name: windows-1809\n    handler: 'containerd-hyperv-17763'\n    scheduling:\n      nodeSelector:\n        kubernetes.io/os: 'windows'\n        kubernetes.io/arch: 'amd64'\n        node.kubernetes.io/windows-build: '10.0.18362'\n      tolerations:\n      - effect: NoSchedule\n        key: windows\n        operator: Equal\n        value: \"true\"\n    ```\n\nAnother RuntimeClass could still be run on the same hosts to use updated containers without Hyper-V isolation.\n\n```yaml\napiVersion: node.k8s.io/v1beta1\nkind: RuntimeClass\nmetadata:\n  name: windows-1903\nhandler: 'default'\nscheduling:\n  nodeSelector:\n    kubernetes.io/os: 'windows'\n    kubernetes.io/arch: 'amd64'\n    node.kubernetes.io/windows-build: '10.0.18362'\n  tolerations:\n  - effect: NoSchedule\n    key: windows\n    operator: Equal\n    value: \"true\"\n```\n\n#### Story 3 - Choosing a specific multi-arch image\n\nStarting from story 2, the RuntimeClass `handler` field could also be used as a means os/version/architecture/variant used by the container runtime. The `handler` is matched up with a corresponding section in the ContainerD configuration file.\n\nHere's an example of what corresponding ContainerD configurations could look like for the runtimes above. The first one would force Hyper-V for compatibility with containers needing 10.0.17763. The second one would be the default for the current version where no Hyper-V isolation is required for compatibility.\n\n```toml\n        [plugins.cri.containerd.runtimes.containerd-hyperv-17763]\n          runtime_type = \"io.containerd.runhcs.v1\"\n          [plugins.cri.containerd.runtimes.containerd-hyperv-17763.options]\n            SandboxImage = \"{{WINDOWSSANDBOXIMAGE}}\"\n            SandboxPlatform = \"windows/amd64\"\n            SandboxOsVersion = \"10.0.17763\"\n            SandboxIsolation = 1\n\n        [plugins.cri.containerd.runtimes.default]\n          runtime_type = \"io.containerd.runhcs.v1\"\n          # No version is specified for process isolation, the node OS version is used\n```\n\n### Implementation Details/Notes/Constraints\n\nThere were multiple options discussed with SIG-Node \u0026 SIG-Windows on October 8 2019 prior to filing this KEP. That discussion \u0026 feedback were captured in [Difficulties in mixed OS and arch clusters]. If you're looking for more details on other approaches excluded, please review that document.\n\n#### Adding new label node.kubernetes.io/windows-build (done)\n\n\u003e Done. This label was added in Kubernetes 1.17\n\nIn [Bounding Self-Labeling Kubelets], a specific range of prefixes were reserved for node self-labeling - `[*.]node.kubernetes.io/*`. Adding a new label within that namespace won't require any changes to NodeRestriction admission. As a new field it also won't require changes to any existing workloads.\n\nBuild numbers will be used instead of product names in the node labels for a few reasons:\n\n- The same OS version may be marketed under two different names due to support / licensing differences. For example, Windows Server 2019 and Windows Server version 1809 are the same build number. The actual binary compatibility is based on build number and the same container can run on either.\n- Windows product names may not have been announced when the Kubernetes project contributors start building and testing against it. Using build numbers instead avoids needing to change Kubernetes source once a name has been announced.\n\nHere are the current product name to build number mappings to illustrate the point:\n\n|Product Name                          |   Build Number(s)      |\n| Windows Server 2019                  | 10.0.17763             |\n| Windows Server version 1809          | 10.0.17763             |\n| Windows Server version 1903          | 10.0.18362             |\n| Windows Server vNext Insider Preview | 10.0.18975, 10.0.18985 |\n\n[Windows Update History] has a full list of version numbers by release \u0026 patch. Starting from an example of `10.0.17763.805` - the OS major, minor, and build number - `10.0.17763` - should match for containers to be compatible. The final `.805` refers to the monthly patches and isn't required for compatibility. Therefore, a value such as `node.kubernetes.io/windows-build = 10.0.17763` will be used. Each one of these Windows Server version will have corresponding containers released - see [Container Base Images] and [Windows Insider Container Images] for details.\n\nThis will pass the regex validation for labels (references: [src1](https://github.com/kubernetes/kubernetes/blob/release-1.16/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go#L30-L32) [src2](https://github.com/kubernetes/kubernetes/blob/release-1.16/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go#L88-L108)). Even the most specific identifier in the Windows registry `BuildLabEx`, for example `18362.1.amd64fre.19h1_release.190318-1202` is within the allowed length and characters to pass the existing validation, although we're not planning to use that whole string. Instead, the kubelet will shorten to just what's needed similar to what's returned today in the output from `kubectl describe node` ([src](https://github.com/kubernetes/kubernetes/blob/0599ca2bcfcae7d702f95284f3c2e2c2978c7772/vendor/github.com/docker/docker/pkg/parsers/operatingsystem/operatingsystem_windows.go#L10))\n\nTo make this easier to consume in the Kubelet and APIs, it will be updated in multiple places:\n\n- Add a well-known label in \n  - [k8s.io/api/core/v1/well_known_labels.go](https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/api/core/v1/well_known_labels.go)\n  - [pkg/kubelet/apis/well_known_labels.go](https://github.com/kubernetes/kubernetes/blob/5a790bce3b222b7fd1ef1225e3b20700c577088a/pkg/kubelet/apis/well_known_labels.go)\n- Set \u0026 update it in Kubelet\n  - [pkg/kubelet/kubelet_node_status.go](https://github.com/kubernetes/kubernetes/blob/4fda1207e347af92e649b59d60d48c7021ba0c54/pkg/kubelet/kubelet_node_status.go#L217)\n\n#### Adding handler to CRI pull API\n\nCRI doesn't have any way to know what os/version/architecture/variant to use when pulling an image today. There are no explicit fields for os/version/architecture/variant in [PodSandboxConfig]\n\nThe same is true for creating a sandbox, but this could be inferred as part of RuntimeHandler. The RuntimeHandler would correspond to a containerd configuration file already on the node which would fill in the missing details.\n\nThe [RunPodSandboxRequest](https://github.com/kubernetes/cri-api/blob/24ae4d4e8b036b885ee1f4930ec2b173eabb28e7/pkg/apis/runtime/v1alpha2/api.proto#L362) message passes the `runtime_handler` as a string.\n\n```\nmessage RunPodSandboxRequest {\n    // Configuration for creating a PodSandbox.\n    PodSandboxConfig config = 1;\n    // Named runtime configuration to use for this PodSandbox.\n    // If the runtime handler is unknown, this request should be rejected.  An\n    // empty string should select the default handler, equivalent to the\n    // behavior before this feature was added.\n    // See https://git.k8s.io/enhancements/keps/sig-node/runtime-class.md\n    string runtime_handler = 2;\n}\n```\n\nThe proposal is to do the same for [PullImageRequest](https://github.com/kubernetes/cri-api/blob/24ae4d4e8b036b885ee1f4930ec2b173eabb28e7/pkg/apis/runtime/v1alpha2/api.proto#L1081). With the additions, it would be:\n\n```\nmessage PullImageRequest {\n    // Spec of the image.\n    ImageSpec image = 1;\n    // Authentication configuration for pulling the image.\n    AuthConfig auth = 2;\n    // Config of the PodSandbox, which is used to pull image in PodSandbox context.\n    PodSandboxConfig sandbox_config = 3;\n    // Named runtime configuration to use for this PodSandbox.\n    // This should match a runtime_handler used in RunPodSandboxRequest\n    // and is subject to the same semantics.\n    string runtime_handler = 4;\n}\n```\n\n### Risks and Mitigations\n\n#### Adding new node label\n\nThe names of aren't part of a versioned API today, so there's no risk to upgrade/downgrade from an API and functionality standpoint. However, if someone wants to keep the node selection experience consistent between Kubernetes 1.14 - 1.17, they may want to manually add the `node.kubernetes.io/windows-build` label to clusters running versions \u003c 1.17. A cluster admin can choose to modify labels using `kubectl label node` after a node has joined the cluster.\n\n#### Adding runtime_handler to PullImageRequest\n\nThis will be reviewed with contributors from both Kubernetes and ContainerD and tested together while Windows support in CRI-ContainerD is still in development. As this is an optional field today in `RunPodSandboxRequest`, there's no risk if someone wants to not specify it in `RuntimeClass`. In that case ContainerD will use the default configuration for pull, create sandbox, and so on.\n\n## Design Details\n\n### Test Plan\n\n#### E2E Testing with CRI-ContainerD and Kubernetes\n\nWe already have E2E tests targeting the CRI-ContainerD \u0026 Kubernetes integration running on Windows - see the KEP for [Windows CRI-ContainerD] for more details. We will add a few `RuntimeClass` definitions to those E2E tests to confirm that Pods can run with `runtime_handler` specified, and without it specified.\n\nAs ContainerD integration itself is an alpha feature, we'll be experimenting with these configurations and deciding on a set of recommended defaults before graduating to beta. These defaults will be tested at beta with new test cases.\n\nThe existing tests relying on `dockershim` will be run side by side until it is deprecated. This timeline hasn't been specified yet, but it will be announced before CRI-ContainerD for Windows is declared stable in 1.19 or later and follow a normal deprecation cycle.\n\n#### Unit testing with CRITest\n\nUnit tests will be added in CRITest which is in the [Cri-Tools] repo. Tests are already running on Windows - see [testgrid](https://k8s-testgrid.appspot.com/sig-node-containerd#cri-validation-windows).\n\n### Graduation Criteria\n\n#### Alpha - Kubernetes 1.17\n\nThis timeline should follow that of [Windows CRI-ContainerD].\n\nIn addition to what's included there, for alpha: \n\n- Unit tests will be added to critest for `runtime_handler`\n- E2E tests for basic coverage of:\n  - new label set by kubelet\n\n#### Alpha -\u003e Beta Graduation\n\nThis timeline should follow that of [Windows CRI-ContainerD].\n\nIn addition to what's included there, for beta:\n\n- Unit tests will be added to critest\n- E2E tests for basic coverage of:\n  - `runtime_handler` updates\n\n#### Beta -\u003e GA Graduation\n\n- Define \u0026 add E2E testing for recommended default RuntimeClass configurations for Windows supported with CRI-ContainerD\n\n\n##### Removing a deprecated flag\n\n- Announce deprecation and support policy of the existing flag\n- Two versions passed since introducing the functionality which deprecates the flag (to address version skew)\n- Address feedback on usage/changed behavior, provided on GitHub issues\n- Deprecate the flag\n\n**For non-optional features moving to GA, the graduation criteria must include [conformance tests].**\n\n[conformance tests]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/conformance-tests.md\n\n### Upgrade / Downgrade Strategy\n\nThe new label `node.kubernetes.io/windows-build` can be set or removed if needed without impacting other components as described in [Risks and Mitigations](#risks-and-mitigations)\n\nUsers can only opt-in to use the new `runtime_handler` field after setting up and configuring ContainerD. On existing clusters without ContainerD set up, they must use `docker` as the `runtimeHandler` in the `RuntimeClass` today. Therefore they must update to a supported version of ContainerD as a prerequisite which is covered in the scope of another KEP - [Windows CRI-ContainerD].\n\n## Implementation History\n\nMajor milestones in the life cycle of a KEP should be tracked in `Implementation History`.\nMajor milestones might include\n\n- the `Summary` and `Motivation` sections being merged signaling SIG acceptance\n- the `Proposal` section being merged signaling agreement on a proposed design\n- the date implementation started\n- the first Kubernetes release where an initial version of the KEP was available\n- the version of Kubernetes where the KEP graduated to general availability\n- when the KEP was retired or superseded\n\n## Alternatives\n\nThese are some other options that were discussed in meetings with SIG-Node and SIG-Windows, but we're not proceeding with. If for some reason `RuntimeClass` support does not graduate to stable or the community prefers to investigate another option, these could serve as starting points for future KEPs.\n\n### Support multiarch os/arch/version in CRI\n\nThe Open Container Initiative specifications for container runtime support specifying the architecture, os, and version when pulling and starting a container. This is important for Windows because there is no kernel compatibility between major versions. In order to successfully start a container with process isolation, the right `os.version` must be pulled and run. Hyper-V can provide backwards compatibility, but the image pull and sandbox creation need to specify `os.version` because the kernel is brought up when the sandbox is created. The same challenges exist for Linux as well because multiple CPU architectures can be supported - for example armv7 with qemu and binfmt_misc.\n\nOne way to make the experience uniform for dealing with multi-arch images is to add new optional fields to force a deployment to use a specific os/version/arch. This may be combined with RuntimeClass to simplify node placement if needed.\n\n```\napiVersion: v1\nkind: Pod\nmetadata:\n  name: mypod\nspec:\n  os: windows\n  osversion: 10.0.17763\n  architecture: amd64\n  runtimeClassName: windows-hyperv\n  containers:\n      - name: iis\n        image: mcr.microsoft.com/windows/servercore/iis:windowsservercore-ltsc2019\n```\n\n\nHere's the steps needed to pull and start a pod+container matching a specific os/arch/version:\n\n- ImageService.PullImage(PullImageRequest) - PullImageRequest includes a `PodSandboxConfig` reference\n- RuntimeService.RunPodSandbox(RunPodSandboxRequest) - RunPodSandboxRequest includes a `PodSandboxConfig` reference\n- RuntimeService.CreateContainer(CreateContainerRequest - the same `PodSandboxConfig` is passed as in the previous step\n\nAll of these use the same `PodSandboxConfig`, so they could be added there.\n\nFrom https://github.com/kubernetes/cri-api/blob/24ae4d4e8b036b885ee1f4930ec2b173eabb28e7/pkg/apis/runtime/v1alpha2/api.proto#L310\n\n```\nmessage PodSandboxConfig {\n    // Metadata of the sandbox. This information will uniquely identify the\n    // sandbox, and the runtime should leverage this to ensure correct\n    // operation. The runtime may also use this information to improve UX, such\n    // as by constructing a readable name.\n    PodSandboxMetadata metadata = 1;\n    // Hostname of the sandbox. Hostname could only be empty when the pod\n    // network namespace is NODE.\n    string hostname = 2;\n    // Path to the directory on the host in which container log files are\n    // stored.\n    // By default the log of a container going into the LogDirectory will be\n    // hooked up to STDOUT and STDERR. However, the LogDirectory may contain\n    // binary log files with structured logging data from the individual\n    // containers. For example, the files might be newline separated JSON\n    // structured logs, systemd-journald journal files, gRPC trace files, etc.\n    // E.g.,\n    //     PodSandboxConfig.LogDirectory = `/var/log/pods/\u003cpodUID\u003e/`\n    //     ContainerConfig.LogPath = `containerName/Instance#.log`\n    //\n    // WARNING: Log management and how kubelet should interface with the\n    // container logs are under active discussion in\n    // https://issues.k8s.io/24677. There *may* be future change of direction\n    // for logging as the discussion carries on.\n    string log_directory = 3;\n    // DNS config for the sandbox.\n    DNSConfig dns_config = 4;\n    // Port mappings for the sandbox.\n    repeated PortMapping port_mappings = 5;\n    // Key-value pairs that may be used to scope and select individual resources.\n    map\u003cstring, string\u003e labels = 6;\n    // Unstructured key-value map that may be set by the kubelet to store and\n    // retrieve arbitrary metadata. This will include any annotations set on a\n    // pod through the Kubernetes API.\n    //\n    // Annotations MUST NOT be altered by the runtime; the annotations stored\n    // here MUST be returned in the PodSandboxStatus associated with the pod\n    // this PodSandboxConfig creates.\n    //\n    // In general, in order to preserve a well-defined interface between the\n    // kubelet and the container runtime, annotations SHOULD NOT influence\n    // runtime behaviour.\n    //\n    // Annotations can also be useful for runtime authors to experiment with\n    // new features that are opaque to the Kubernetes APIs (both user-facing\n    // and the CRI). Whenever possible, however, runtime authors SHOULD\n    // consider proposing new typed fields for any new features instead.\n    map\u003cstring, string\u003e annotations = 7;\n    // Optional configurations specific to Linux hosts.\n    LinuxPodSandboxConfig linux = 8;\n}\n```\n\nToday, PodSandboxConfig has an annotation that can be used as a workaround, but it's stated that \"Whenever possible, however, runtime authors SHOULD consider proposing new typed fields for any new features instead.\".\n\nThe proposed additions are:\n\n```\nstring os = 9;\nstring architecture = 10;\nstring version = 11;\n```\n\nThese correspond to `platform.os`, `platform.architecture`, and `platform.os.version` as describe in the [OCI image spec](https://github.com/opencontainers/image-spec/blob/master/image-index.md)\n\n### Make the scheduler aware of Multi-arch images\n\nWhen scheduling a pod, it would need to retrieve the container manifest to infer what os, version and architecture was intended. This could be used to find a matching node and avoid the pull failure. However, this was previously rejected in discussions with SIG-Architecture and SIG-Scheduling because it would impact scheduler performance. Additionally, the scheduler makes no sync network calls to other services (such as a container registry) to make scheduling decisions. It can run in isolation and only connects to other trusted pods such as the APIServer.\n\n### Create a multi-arch Mutating admission controller\n\nBefore a deployment is scheduled, it can be handed off to a mutating admission controller registered with a webhook. This could do extra work such as a manifest pull \u0026 analysis, then add additional info to the deployment such as a NodeSelector or RuntimeClass. The behavior would probably need to be configurable and tailored to a customer's deployment. To round off the experience, it could also reject requests that can't be fulfilled (no matching OS / architecture) immediately.\n\nBased on a deployment, infer and determine the extra info needed that needs to be passed to ContainerD. This could work within the existing APIs (RuntimeClass / NodeSelector) or work with extended APIs (annotations or more specific pod API).\n\nThis approach would still impact scheduling latency if a pod has omitted any of the NodeSelector required by the heuristic. The synchronous calls to a container registry are still needed.\n\n## Future Considerations\n\nWhile this feature is in alpha, we can continue experimenting with the user experience. There are other enhancements that may work well with these changes to provide an easier experience.\n\n### Pod Overhead\n\nThe [Pod overhead KEP] proposes adding an `Overhead` field using the same `ContainerResources` structure today. This would make the scheduler aware of how much resources are required by the container runtime, such as ContainerD with Hyper-V isolation, in addition to those requested for the Pod itself. If this KEP is implemented, it can be tested with multiple hypervisors and used to help build consistency across them.\n\n### RuntimeClass Parameters\n\nRuntimeClass aims to replace experimental pod annotations with a [Runtime Handler] instead. Today that doesn't include any CRI-specific configuration that's passed through CRI. However, if there's a clear use case and need to consolidate parameters across multiple sandbox implementations, it could be added. One easy example might be setting the cpu or memory topology presented from the hypervisor to the sandbox. If a consensus emerges between multiple sandboxed CRI providers such as Kata, Firecracker \u0026 Hyper-V, then RuntimeClass could be updated to include more standard parameters and replace some configurations kept in separate configuration files today.\n\n## Reference \u0026 Examples\n\n### Multi-arch container image overview\n\nThe Open Container Initiative specifications for container runtime support specifying the architecture, os, and version when pulling and starting a container. \n\nHere's one example of a container image that has a multi-arch manifest with entries for Linux \u0026 Windows, multiple CPU architectures, and multiple Windows os versions.\n\n```powershell\n(docker manifest inspect golang:latest | ConvertFrom-Json)[0].manifests | Format-Table digest, platform\n\ndigest                                                                  platform\nsha256:a50a9364e9170ab5f5b03389ed33b9271b4a7b6bbb0ab41c4035adb3078927bc @{architecture=amd64; os=linux}\nsha256:30526a829a37fe2ba8231c06142879f7f6873bc6ebe78bc99674f8ea0e111815 @{architecture=arm; os=linux; variant=v7}\nsha256:a05d345bf4635df552ce9635708676c607d2b833278396470bf5788eea0a4b1c @{architecture=arm64; os=linux; variant=v8}\nsha256:b11bad2ef5ef90ab7e5589d9a5af51bc3f65335278e73f95b18db2057c0505ae @{architecture=386; os=linux}\nsha256:a7db5fe778800809dc1cacd6ae4a1c33ce3f4eb8f39d722b358d7fb27b3a1f1c @{architecture=ppc64le; os=linux}\nsha256:a0a8410be5cb7970e00d98dff42e26afad237c08a746cdf375a1b1ad3e4df08c @{architecture=s390x; os=linux}\nsha256:a6bf1ef2d20ecbf73d5d1729182a37377bd8820a0871a0422f27bcad6b928d76 @{architecture=amd64; os=windows; os.version=10.0.14393.3274}\nsha256:5141a4422a77e493d48012f65f35c413f4d4ca7da5f450d96227b0c15b3de3e8 @{architecture=amd64; os=windows; os.version=10.0.17134.1069}\nsha256:d22e5bf156af4df25a24cb268e955df3503cd91b50cd43b9bcf4bccf7a3c0804 @{architecture=amd64; os=windows; os.version=10.0.17763.805}\n```\n\n[Windows CRI-ContainerD]: /keps/sig-windows/20190424-windows-cri-containerd.md\n[Guide for scheduling Windows containers in Kubernetes]: https://kubernetes.io/docs/setup/production-environment/windows/user-guide-windows-containers/#taints-and-tolerations\n[RuntimeClass Scheduling]: https://kubernetes.io/docs/concepts/containers/runtime-class/#scheduling\n[Gatekeeper]: https://github.com/open-policy-agent/gatekeeper\n[Difficulties in mixed OS and arch clusters]: https://docs.google.com/document/d/12uZt-KSG8v4CSyUDr0EC6btmzpVOZAWzqYDif3EoeBU/edit#\n[PodSandboxConfig]: https://github.com/kubernetes/cri-api/blob/24ae4d4e8b036b885ee1f4930ec2b173eabb28e7/pkg/apis/runtime/v1alpha2/api.proto#L310\n[Bounding Self-Labeling Kubelets]: https://github.com/kubernetes/enhancements/blob/f1a799d5f4658ed29797c1fb9ceb7a4d0f538e93/keps/sig-auth/0000-20170814-bounding-self-labeling-kubelets.md\n[Windows Update History]: https://support.microsoft.com/en-us/help/4498140\n[Cri-Tools]: https://github.com/kubernetes-sigs/cri-tools\n[Windows container version compatibility]: https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/version-compatibility\n[Pod overhead KEP]: https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/20190226-pod-overhead.md#container-runtime-interface-cri\n[Runtime Handler]: https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/runtime-class.md#runtime-handler\n[Container Base Images]: https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/container-base-images\n[Windows Insider Container Images]: https://docs.microsoft.com/en-us/virtualization/windowscontainers/quick-start/using-insider-container-images#install-base-container-image\n"
+  }
+]
diff --git a/src/assets/release-notes-1.14.0.json b/src/assets/release-notes-1.14.0.json
deleted file mode 100644
index 8959f36a..00000000
--- a/src/assets/release-notes-1.14.0.json
+++ /dev/null
@@ -1,3233 +0,0 @@
-{
-  "59176": {
-    "commit": "73ccf56edd86924a6043f8b682777854a5d79efd",
-    "text": "exit kube-proxy when configuration file changes",
-    "markdown": "exit kube-proxy when configuration file changes ([#59176](https://github.com/kubernetes/kubernetes/pull/59176), [@dixudx](https://github.com/dixudx))",
-    "author": "dixudx",
-    "author_url": "https://github.com/dixudx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/59176",
-    "pr_number": 59176,
-    "sigs": ["cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "63947": {
-    "commit": "d532d50e29b887d34021e4cc2330e41da005ca12",
-    "text": "New conformance tests added for API Aggregation.",
-    "markdown": "New conformance tests added for API Aggregation. ([#63947](https://github.com/kubernetes/kubernetes/pull/63947), [@jennybuckley](https://github.com/jennybuckley))\n\n  Courtesy of SIG API Machinery, SIG Architecture, and SIG Testing",
-    "author": "jennybuckley",
-    "author_url": "https://github.com/jennybuckley",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/63947",
-    "pr_number": 63947,
-    "areas": ["conformance"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "architecture", "testing"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "64820": {
-    "commit": "a92729a301c8928d8e108418e6e4625a9e0d6733",
-    "text": "Add `kubectl auth can-i --list` option which could help users know what actions they can do in specific namespace.",
-    "markdown": "Add `kubectl auth can-i --list` option which could help users know what actions they can do in specific namespace. ([#64820](https://github.com/kubernetes/kubernetes/pull/64820), [@WanLinghao](https://github.com/WanLinghao))",
-    "author": "WanLinghao",
-    "author_url": "https://github.com/WanLinghao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/64820",
-    "pr_number": 64820,
-    "areas": ["kubectl"],
-    "sigs": ["auth", "cli"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "65636": {
-    "commit": "836ce9fdc13d0ecfcfdfb7ba4b713ac092ee16bc",
-    "text": "Fixed panic when performing a \"set env\" operation on a --local resource",
-    "markdown": "Fixed panic when performing a \"set env\" operation on a --local resource ([#65636](https://github.com/kubernetes/kubernetes/pull/65636), [@juanvallejo](https://github.com/juanvallejo))",
-    "author": "juanvallejo",
-    "author_url": "https://github.com/juanvallejo",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/65636",
-    "pr_number": 65636,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.0"
-  },
-  "67350": {
-    "commit": "9059021d9ad9c988672f7663856f9c1b4c90fd01",
-    "text": "watch.Until now works for long durations.",
-    "markdown": "watch.Until now works for long durations. ([#67350](https://github.com/kubernetes/kubernetes/pull/67350), [@tnozicka](https://github.com/tnozicka))",
-    "author": "tnozicka",
-    "author_url": "https://github.com/tnozicka",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/67350",
-    "pr_number": 67350,
-    "areas": ["apiserver", "deflake", "kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth", "cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "67573": {
-    "commit": "1ddfd8ff73660934801059e3c89f31cdbbbc80c8",
-    "text": "It is now possible to combine the `-f` and `-l` flags in `kubectl logs`",
-    "markdown": "It is now possible to combine the `-f` and `-l` flags in `kubectl logs` ([#67573](https://github.com/kubernetes/kubernetes/pull/67573), [@m1kola](https://github.com/m1kola))\n\n  Courtesy of SIG CLI, and SIG Scalability",
-    "author": "m1kola",
-    "author_url": "https://github.com/m1kola",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/67573",
-    "pr_number": 67573,
-    "areas": ["kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["cli", "scalability"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "67678": {
-    "commit": "0ae81c986aef7e12b5746087aa9359b88efb456e",
-    "text": "The --storage-versions flag of kube-apiserver is removed. The storage versions will always be the default value built-in the kube-apiserver binary.",
-    "markdown": "The --storage-versions flag of kube-apiserver is removed. The storage versions will always be the default value built-in the kube-apiserver binary. ([#67678](https://github.com/kubernetes/kubernetes/pull/67678), [@caesarxuchao](https://github.com/caesarxuchao))\n\n  Courtesy of SIG API Machinery, and SIG Testing",
-    "author": "caesarxuchao",
-    "author_url": "https://github.com/caesarxuchao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/67678",
-    "pr_number": 67678,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "testing"],
-    "action_required": true,
-    "release_version": "1.14.0"
-  },
-  "68925": {
-    "commit": "f3a6dbceb21fc51666f16826c9db8ee28b3aa300",
-    "text": "Fixes a bug that prevented deletion of dynamically provisioned volumes in Quobyte backends.",
-    "markdown": "Fixes a bug that prevented deletion of dynamically provisioned volumes in Quobyte backends. ([#68925](https://github.com/kubernetes/kubernetes/pull/68925), [@casusbelli](https://github.com/casusbelli))",
-    "author": "casusbelli",
-    "author_url": "https://github.com/casusbelli",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/68925",
-    "pr_number": 68925,
-    "kinds": ["api-change"],
-    "sigs": ["apps", "architecture", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "69099": {
-    "commit": "2f2945732b5821f9d51c6acdfb0e9fabe81bf3a4",
-    "text": "Added cadvisor metric labels `pod` and `container` where `pod_name` and `container_name` are present to match instrumentation guidelines.\n\nany Prometheus queries that match `pod_name` and `container_name` labels (e.g. cadvisor or kubelet probe metrics) should be updated to use `pod` and `container` instead. `pod_name` and `container_name` labels will be present alongside `pod` and `container` labels for one transitional release and removed in the future.",
-    "markdown": "Added cadvisor metric labels `pod` and `container` where `pod_name` and `container_name` are present to match instrumentation guidelines.\n  \n  any Prometheus queries that match `pod_name` and `container_name` labels (e.g. cadvisor or kubelet probe metrics) should be updated to use `pod` and `container` instead. `pod_name` and `container_name` labels will be present alongside `pod` and `container` labels for one transitional release and removed in the future. ([#69099](https://github.com/kubernetes/kubernetes/pull/69099), [@ehashman](https://github.com/ehashman))\n\n  Courtesy of SIG Instrumentation, and SIG Node",
-    "author": "ehashman",
-    "author_url": "https://github.com/ehashman",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/69099",
-    "pr_number": 69099,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["instrumentation", "node"],
-    "action_required": true,
-    "release_version": "1.14.0"
-  },
-  "69255": {
-    "commit": "2cd88258c75527b6ede2257149fb4a01b552d831",
-    "text": "Remove deprecated args '--show-all'",
-    "markdown": "Remove deprecated args '--show-all' ([#69255](https://github.com/kubernetes/kubernetes/pull/69255), [@Pingan2017](https://github.com/Pingan2017))",
-    "author": "Pingan2017",
-    "author_url": "https://github.com/Pingan2017",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/69255",
-    "pr_number": 69255,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.14.0"
-  },
-  "69279": {
-    "commit": "9cc39a5614d8b5d8c9b750c0c737c1a58bde18bf",
-    "text": "Print `SizeLimit` of `EmptyDir` in `kubectl describe pod` outputs.",
-    "markdown": "Print `SizeLimit` of `EmptyDir` in `kubectl describe pod` outputs. ([#69279](https://github.com/kubernetes/kubernetes/pull/69279), [@dtaniwaki](https://github.com/dtaniwaki))",
-    "author": "dtaniwaki",
-    "author_url": "https://github.com/dtaniwaki",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/69279",
-    "pr_number": 69279,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.14.0"
-  },
-  "69366": {
-    "commit": "b66e332d3c19ce78e00dd7c904fa29e8f6784ba0",
-    "text": "kubeadm now attempts to detect an installed CRI by its usual domain socket, so that --cri-socket can be omitted from the command line if Docker is not used and there is a single CRI installed.",
-    "markdown": "kubeadm now attempts to detect an installed CRI by its usual domain socket, so that --cri-socket can be omitted from the command line if Docker is not used and there is a single CRI installed. ([#69366](https://github.com/kubernetes/kubernetes/pull/69366), [@rosti](https://github.com/rosti))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/69366",
-    "pr_number": 69366,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "69552": {
-    "commit": "6af7d2cb1a2dd0d9ab7c3c9f2d9650959e4533d4",
-    "text": "remove kubelet flag '--experimental-fail-swap-on' (deprecated in v1.8)",
-    "markdown": "remove kubelet flag '--experimental-fail-swap-on' (deprecated in v1.8) ([#69552](https://github.com/kubernetes/kubernetes/pull/69552), [@Pingan2017](https://github.com/Pingan2017))",
-    "author": "Pingan2017",
-    "author_url": "https://github.com/Pingan2017",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/69552",
-    "pr_number": 69552,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.14.0"
-  },
-  "69782": {
-    "commit": "23a085062c9dc531ec601210c69d65e25e43e6aa",
-    "text": "StorageOS volume plugin updated to fix an issue where volume mount succeeds even if request to mount via StorageOS API fails.",
-    "markdown": "StorageOS volume plugin updated to fix an issue where volume mount succeeds even if request to mount via StorageOS API fails. ([#69782](https://github.com/kubernetes/kubernetes/pull/69782), [@darkowlzz](https://github.com/darkowlzz))",
-    "author": "darkowlzz",
-    "author_url": "https://github.com/darkowlzz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/69782",
-    "pr_number": 69782,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.0"
-  },
-  "69940": {
-    "commit": "baaaa15fdf86a1e2f211f525676996a484bb3903",
-    "text": "CoreDNS is only officially supported on Linux at this time.  As such, when kubeadm is used to deploy this component into your kubernetes cluster, it will be restricted (using nodeSelectors) to run only on nodes with that operating system. This ensures that in clusters which include Windows nodes, the scheduler will not ever attempt to place CoreDNS pods on these machines, reducing setup latency and enhancing initial cluster stability.",
-    "markdown": "CoreDNS is only officially supported on Linux at this time.  As such, when kubeadm is used to deploy this component into your kubernetes cluster, it will be restricted (using nodeSelectors) to run only on nodes with that operating system. This ensures that in clusters which include Windows nodes, the scheduler will not ever attempt to place CoreDNS pods on these machines, reducing setup latency and enhancing initial cluster stability. ([#69940](https://github.com/kubernetes/kubernetes/pull/69940), [@MarcPow](https://github.com/MarcPow))",
-    "author": "MarcPow",
-    "author_url": "https://github.com/MarcPow",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/69940",
-    "pr_number": 69940,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "70121": {
-    "commit": "697c2316faaabae8ef8371032b60be65d7795e68",
-    "text": "Add network stats for Windows nodes and pods.",
-    "markdown": "Add network stats for Windows nodes and pods. ([#70121](https://github.com/kubernetes/kubernetes/pull/70121), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/70121",
-    "pr_number": 70121,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "70144": {
-    "commit": "d95b9f16d1bb84534efb609656883224a7110915",
-    "text": "Enable mTLS encription between etcd and kube-apiserver in GCE",
-    "markdown": "Enable mTLS encription between etcd and kube-apiserver in GCE ([#70144](https://github.com/kubernetes/kubernetes/pull/70144), [@wenjiaswe](https://github.com/wenjiaswe))\n\n  Courtesy of SIG API Machinery, and SIG Cluster Lifecycle",
-    "author": "wenjiaswe",
-    "author_url": "https://github.com/wenjiaswe",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/70144",
-    "pr_number": 70144,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "70309": {
-    "commit": "b0a455b8119b15bad1ddfe2c67fca5b212163740",
-    "text": "AWS ELB health checks will now use HTTPS/SSL protocol for HTTPS/SSL backends.",
-    "markdown": "AWS ELB health checks will now use HTTPS/SSL protocol for HTTPS/SSL backends. ([#70309](https://github.com/kubernetes/kubernetes/pull/70309), [@2rs2ts](https://github.com/2rs2ts))",
-    "author": "2rs2ts",
-    "author_url": "https://github.com/2rs2ts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/70309",
-    "pr_number": 70309,
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.0"
-  },
-  "70647": {
-    "commit": "cf24d24e6641096be9356e812b09fe637a2f606b",
-    "text": "Image garbage collection no longer fails for images with only one tag but more than one repository associated.",
-    "markdown": "Image garbage collection no longer fails for images with only one tag but more than one repository associated. ([#70647](https://github.com/kubernetes/kubernetes/pull/70647), [@corvus-ch](https://github.com/corvus-ch))",
-    "author": "corvus-ch",
-    "author_url": "https://github.com/corvus-ch",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/70647",
-    "pr_number": 70647,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.0"
-  },
-  "70805": {
-    "commit": "56e88f5b47da3dfc0a95e0bc9be6a0c6c3275345",
-    "text": "Fixes the setting of NodeAddresses when using the vSphere CloudProvider and nodes that have multiple IP addresses.",
-    "markdown": "Fixes the setting of NodeAddresses when using the vSphere CloudProvider and nodes that have multiple IP addresses. ([#70805](https://github.com/kubernetes/kubernetes/pull/70805), [@danwinship](https://github.com/danwinship))",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/70805",
-    "pr_number": 70805,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.0"
-  },
-  "70896": {
-    "commit": "c06ca11b5dc637fe3df3fb2b50a1980e1237654c",
-    "text": "Added kube-proxy support for overlay networking and DSR in Windows and new flags for network-name, source-vip, and enable-dsr.",
-    "markdown": "Added kube-proxy support for overlay networking and DSR in Windows and new flags for network-name, source-vip, and enable-dsr. ([#70896](https://github.com/kubernetes/kubernetes/pull/70896), [@ksubrmnn](https://github.com/ksubrmnn))",
-    "author": "ksubrmnn",
-    "author_url": "https://github.com/ksubrmnn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/70896",
-    "pr_number": 70896,
-    "areas": ["apiserver", "kubeadm"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "network", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "71117": {
-    "commit": "9d58c8fbcff3ab6c858df5b7ed557f162381ff61",
-    "text": "kubectl loads config file once and uses persistent client config",
-    "markdown": "kubectl loads config file once and uses persistent client config ([#71117](https://github.com/kubernetes/kubernetes/pull/71117), [@dixudx](https://github.com/dixudx))",
-    "author": "dixudx",
-    "author_url": "https://github.com/dixudx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/71117",
-    "pr_number": 71117,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.0"
-  },
-  "71147": {
-    "commit": "e4db1471f11930b97fdeaf6ce0b9feeca243d035",
-    "text": "vSphere cloud provider correctly retrieves the VM's UUID when running on Windows",
-    "markdown": "vSphere cloud provider correctly retrieves the VM's UUID when running on Windows ([#71147](https://github.com/kubernetes/kubernetes/pull/71147), [@benmoss](https://github.com/benmoss))\n\n  Courtesy of SIG Cloud Provider, and SIG Windows",
-    "author": "benmoss",
-    "author_url": "https://github.com/benmoss",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/71147",
-    "pr_number": 71147,
-    "areas": ["provider/vmware"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider", "windows"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "71192": {
-    "commit": "0b48018a39c26f86c4f04fe0b5544d7624d4495f",
-    "text": "kube-apiserver now serves OpenAPI specs for registered CRDs with defined \nvalidation schemata as an alpha feature, to be enabled via the \"CustomResourcePublishOpenAPI\" feature gate. Kubectl will validate client-side using those. Note that in\nfuture, client-side validation in 1.14 kubectl against a 1.15 cluster will reject\nunknown fields for CRDs with validation schema defined.",
-    "markdown": "kube-apiserver now serves OpenAPI specs for registered CRDs with defined \n  validation schemata as an alpha feature, to be enabled via the \"CustomResourcePublishOpenAPI\" feature gate. Kubectl will validate client-side using those. Note that in\n  future, client-side validation in 1.14 kubectl against a 1.15 cluster will reject\n  unknown fields for CRDs with validation schema defined. ([#71192](https://github.com/kubernetes/kubernetes/pull/71192), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/71192",
-    "pr_number": 71192,
-    "areas": ["apiserver", "custom-resources", "kubectl", "release-eng"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "architecture", "cli", "release", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "71223": {
-    "commit": "6912bbb153ef53a0ece34e7dae74ca79bfc07b82",
-    "text": "Considerably reduced the CPU load in kube-apiserver while aggregating OpenAPI specifications from aggregated API servers.",
-    "markdown": "Considerably reduced the CPU load in kube-apiserver while aggregating OpenAPI specifications from aggregated API servers. ([#71223](https://github.com/kubernetes/kubernetes/pull/71223), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/71223",
-    "pr_number": 71223,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "71351": {
-    "commit": "5bfea15e7b8bde1a9bb891513d8d0c9812ab8358",
-    "text": "Extends the VolumeSubpathEnvExpansion alpha feature to support environment variable expansion\nImplements subPathExpr field for expanding environment variables into a subPath\nThe fields subPathExpr and subPath are mutually exclusive\nNote: This is a breaking change from the previous version of this alpha feature",
-    "markdown": "Extends the VolumeSubpathEnvExpansion alpha feature to support environment variable expansion\n  Implements subPathExpr field for expanding environment variables into a subPath\n  The fields subPathExpr and subPath are mutually exclusive\n  Note: This is a breaking change from the previous version of this alpha feature ([#71351](https://github.com/kubernetes/kubernetes/pull/71351), [@kevtaylor](https://github.com/kevtaylor))",
-    "author": "kevtaylor",
-    "author_url": "https://github.com/kevtaylor",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/71351",
-    "pr_number": 71351,
-    "areas": ["apiserver", "kubelet"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "apps", "architecture", "node", "storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "71479": {
-    "commit": "55a65763c03b21029017fc12a422e5dbff9d4455",
-    "text": "Adds the same information to an init container as a standard container in a pod when using PodPresets.",
-    "markdown": "Adds the same information to an init container as a standard container in a pod when using PodPresets. ([#71479](https://github.com/kubernetes/kubernetes/pull/71479), [@soggiest](https://github.com/soggiest))\n\n  Courtesy of SIG Node, SIG Service Catalog, and SIG Testing",
-    "author": "soggiest",
-    "author_url": "https://github.com/soggiest",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/71479",
-    "pr_number": 71479,
-    "kinds": ["feature"],
-    "sigs": ["node", "service-catalog", "testing"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "71548": {
-    "commit": "6f9bf5fe98bcc3b436fea4d6dd345a1502d20778",
-    "text": "Watch will now support converting response objects into Table or PartialObjectMetadata forms.",
-    "markdown": "Watch will now support converting response objects into Table or PartialObjectMetadata forms. ([#71548](https://github.com/kubernetes/kubernetes/pull/71548), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/71548",
-    "pr_number": 71548,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "71651": {
-    "commit": "14b671b24be001895f7ca073f7d6ade8c5bc5182",
-    "text": "add subcommand `kubectl create cronjob`",
-    "markdown": "add subcommand `kubectl create cronjob` ([#71651](https://github.com/kubernetes/kubernetes/pull/71651), [@Pingan2017](https://github.com/Pingan2017))\n\n  Courtesy of SIG CLI",
-    "author": "Pingan2017",
-    "author_url": "https://github.com/Pingan2017",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/71651",
-    "pr_number": 71651,
-    "areas": ["kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "71767": {
-    "commit": "fcbe7f50f26579f97aaa9cbe7ed0b8aacc694df4",
-    "text": "Prometheus metrics for crd_autoregister, crd_finalizer and crd_naming_condition_controller are exported.",
-    "markdown": "Prometheus metrics for crd_autoregister, crd_finalizer and crd_naming_condition_controller are exported. ([#71767](https://github.com/kubernetes/kubernetes/pull/71767), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/71767",
-    "pr_number": 71767,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.0"
-  },
-  "71874": {
-    "commit": "37f0138278809ec3245de8e7b861b7bf9d323b45",
-    "text": "kubeadm: fix a bug where standard kubeconfig paths were searched even if the user provided /etc/kubernetes/admin.conf explicitly for commands that accept --kubeconfig, like kubeadm token.\nkubeadm: use the default kubeconfig (/etc/kubernetes/admin.conf) for \"kubeadm reset\" and \"kubeadm upgrade\" commands.",
-    "markdown": "kubeadm: fix a bug where standard kubeconfig paths were searched even if the user provided /etc/kubernetes/admin.conf explicitly for commands that accept --kubeconfig, like kubeadm token.\n  kubeadm: use the default kubeconfig (/etc/kubernetes/admin.conf) for \"kubeadm reset\" and \"kubeadm upgrade\" commands. ([#71874](https://github.com/kubernetes/kubernetes/pull/71874), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/71874",
-    "pr_number": 71874,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "72062": {
-    "commit": "0713f29c289a339fef06647c10fa65d59a169a8d",
-    "text": "Reduce GCE log rotation check from 1 hour to every 5 minutes.  Rotation policy is unchanged (new day starts, log file size \u003e 100MB).",
-    "markdown": "Reduce GCE log rotation check from 1 hour to every 5 minutes.  Rotation policy is unchanged (new day starts, log file size \u003e 100MB). ([#72062](https://github.com/kubernetes/kubernetes/pull/72062), [@jpbetz](https://github.com/jpbetz))",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72062",
-    "pr_number": 72062,
-    "areas": ["platform/gce"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "72214": {
-    "commit": "d8f014613865955b7e4ce6fefbf38a3c8fe97971",
-    "text": "Breaking changes in client-go:\nThe disk-cached discovery client is moved from k8s.io/client-go/discovery to k8s.io/client-go/discovery/cached/disk.\nThe memory-cached discovery client is moved from k8s.io/client-go/discovery/cached to k8s.io/client-go/discovery/cached/memory.",
-    "markdown": "Breaking changes in client-go:\n  The disk-cached discovery client is moved from k8s.io/client-go/discovery to k8s.io/client-go/discovery/cached/disk.\n  The memory-cached discovery client is moved from k8s.io/client-go/discovery/cached to k8s.io/client-go/discovery/cached/memory. ([#72214](https://github.com/kubernetes/kubernetes/pull/72214), [@caesarxuchao](https://github.com/caesarxuchao))",
-    "author": "caesarxuchao",
-    "author_url": "https://github.com/caesarxuchao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72214",
-    "pr_number": 72214,
-    "areas": ["apiserver", "kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "72245": {
-    "commit": "26d32a7cffd7abfaaa88823b492d218f2289a1ea",
-    "text": "Adds configuration for AWS endpoint fine control:\nOverrideEndpoints bool Set to true to allow custom endpoints\nServiceDelimiter string Delimiter to use to separate overridden services (multiple services) Defaults to \"\u0026\"\nServicenameDelimiter string Delimiter to use to separate servicename from its configuration parameters Defaults \"|\"\nOverrideSeparator string Delimiter to use to separate region of occurrence, url and signing region for each override Defaults to \",\"\nServiceOverrides string example: s3|region1, https://s3.foo.bar, some signing_region \u0026 ec2|region2, https://ec2.foo.bar, signing_region",
-    "markdown": "Adds configuration for AWS endpoint fine control:\n  OverrideEndpoints bool Set to true to allow custom endpoints\n  ServiceDelimiter string Delimiter to use to separate overridden services (multiple services) Defaults to \"\u0026\"\n  ServicenameDelimiter string Delimiter to use to separate servicename from its configuration parameters Defaults \"|\"\n  OverrideSeparator string Delimiter to use to separate region of occurrence, url and signing region for each override Defaults to \",\"\n  ServiceOverrides string example: s3|region1, https://s3.foo.bar, some signing_region \u0026 ec2|region2, https://ec2.foo.bar, signing_region ([#72245](https://github.com/kubernetes/kubernetes/pull/72245), [@ampsingram](https://github.com/ampsingram))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "ampsingram",
-    "author_url": "https://github.com/ampsingram",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72245",
-    "pr_number": 72245,
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "72286": {
-    "commit": "c336229b58de48a6c8c456e3347f077ace97f7ea",
-    "text": "e2e tests that require SSH may be used against clusters that have nodes without external IP addresses by setting the environment variable `KUBE_SSH_BASTION` to the `host:port` of a machine that is allowed to SSH to those nodes.  The same private key that the test would use is used for the bastion host.  The test connects to the bastion and then tunnels another SSH connection to the node.",
-    "markdown": "e2e tests that require SSH may be used against clusters that have nodes without external IP addresses by setting the environment variable `KUBE_SSH_BASTION` to the `host:port` of a machine that is allowed to SSH to those nodes.  The same private key that the test would use is used for the bastion host.  The test connects to the bastion and then tunnels another SSH connection to the node. ([#72286](https://github.com/kubernetes/kubernetes/pull/72286), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72286",
-    "pr_number": 72286,
-    "kinds": ["bug"],
-    "sigs": ["node", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "72323": {
-    "commit": "ae450686883774737ae646a2d210f8b7ff5249b8",
-    "text": "Change docker metrics to conform metrics guidelines and using histogram for better aggregation.\nThe following metrics are deprecated, and will be removed in a future release:\n- `docker_operations`\n- `docker_operations_latency_microseconds`\n- `docker_operations_errors`\n- `docker_operations_timeout`\n- `network_plugin_operations_latency_microseconds`\nPlease convert to the following metrics:\n- `docker_operations_total`\n- `docker_operations_latency_seconds`\n- `docker_operations_errors_total`\n- `docker_operations_timeout_total`\n- `network_plugin_operations_latency_seconds`",
-    "markdown": "Change docker metrics to conform metrics guidelines and using histogram for better aggregation.\n  The following metrics are deprecated, and will be removed in a future release:\n  - `docker_operations`\n  - `docker_operations_latency_microseconds`\n  - `docker_operations_errors`\n  - `docker_operations_timeout`\n  - `network_plugin_operations_latency_microseconds`\n  Please convert to the following metrics:\n  - `docker_operations_total`\n  - `docker_operations_latency_seconds`\n  - `docker_operations_errors_total`\n  - `docker_operations_timeout_total`\n  - `network_plugin_operations_latency_seconds` ([#72323](https://github.com/kubernetes/kubernetes/pull/72323), [@danielqsj](https://github.com/danielqsj))\n\n  Courtesy of SIG Instrumentation, SIG Node, and SIG Testing",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72323",
-    "pr_number": 72323,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["instrumentation", "node", "testing"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "72334": {
-    "commit": "b8d6de320f302235a59ab90e7c5d4826e27673bc",
-    "text": "Change proxy metrics to conform metrics guidelines.\nThe metrics `sync_proxy_rules_latency_microseconds` is deprecated, and will be removed in a future release, please convert to metrics`sync_proxy_rules_latency_seconds`.",
-    "markdown": "Change proxy metrics to conform metrics guidelines.\n  The metrics `sync_proxy_rules_latency_microseconds` is deprecated, and will be removed in a future release, please convert to metrics`sync_proxy_rules_latency_seconds`. ([#72334](https://github.com/kubernetes/kubernetes/pull/72334), [@danielqsj](https://github.com/danielqsj))\n\n  Courtesy of SIG Instrumentation, and SIG Network",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72334",
-    "pr_number": 72334,
-    "kinds": ["feature"],
-    "sigs": ["instrumentation", "network"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "72336": {
-    "commit": "24643fd1163857a507e40f4bceb303c5b44d2d8f",
-    "text": "Change apiserver metrics to conform metrics guidelines.\nThe following metrics are deprecated, and will be removed in a future release:\n- `apiserver_request_count`\n- `apiserver_request_latencies`\n- `apiserver_request_latencies_summary`\n- `apiserver_dropped_requests`\n- `etcd_helper_cache_hit_count`\n- `etcd_helper_cache_miss_count`\n- `etcd_helper_cache_entry_count`\n- `etcd_request_cache_get_latencies_summary`\n- `etcd_request_cache_add_latencies_summary`\n- `etcd_request_latencies_summary`\n- `transformation_latencies_microseconds `\n- `data_key_generation_latencies_microseconds`\nPlease convert to the following metrics:\n- `apiserver_request_total`\n- `apiserver_request_latency_seconds`\n- `apiserver_dropped_requests_total`\n- `etcd_helper_cache_hit_total`\n- `etcd_helper_cache_miss_total`\n- `etcd_helper_cache_entry_total`\n- `etcd_request_cache_get_latency_seconds`\n- `etcd_request_cache_add_latency_seconds`\n- `etcd_request_latency_seconds`\n- `transformation_latencies_seconds`\n- `data_key_generation_latencies_seconds`",
-    "markdown": "Change apiserver metrics to conform metrics guidelines.\n  The following metrics are deprecated, and will be removed in a future release:\n  - `apiserver_request_count`\n  - `apiserver_request_latencies`\n  - `apiserver_request_latencies_summary`\n  - `apiserver_dropped_requests`\n  - `etcd_helper_cache_hit_count`\n  - `etcd_helper_cache_miss_count`\n  - `etcd_helper_cache_entry_count`\n  - `etcd_request_cache_get_latencies_summary`\n  - `etcd_request_cache_add_latencies_summary`\n  - `etcd_request_latencies_summary`\n  - `transformation_latencies_microseconds `\n  - `data_key_generation_latencies_microseconds`\n  Please convert to the following metrics:\n  - `apiserver_request_total`\n  - `apiserver_request_latency_seconds`\n  - `apiserver_dropped_requests_total`\n  - `etcd_helper_cache_hit_total`\n  - `etcd_helper_cache_miss_total`\n  - `etcd_helper_cache_entry_total`\n  - `etcd_request_cache_get_latency_seconds`\n  - `etcd_request_cache_add_latency_seconds`\n  - `etcd_request_latency_seconds`\n  - `transformation_latencies_seconds`\n  - `data_key_generation_latencies_seconds` ([#72336](https://github.com/kubernetes/kubernetes/pull/72336), [@danielqsj](https://github.com/danielqsj))\n\n  Courtesy of SIG API Machinery, SIG Instrumentation, and SIG Testing",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72336",
-    "pr_number": 72336,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "instrumentation", "testing"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "72343": {
-    "commit": "035332f1ab664c1a2dc2b1144b7a0a7f1f10f46d",
-    "text": "Fix admission metrics in seconds.\nAdd metrics `*_admission_latencies_milliseconds` and `*_admission_latencies_milliseconds_summary` for backward compatible, but will be removed in a future release.",
-    "markdown": "Fix admission metrics in seconds.\n  Add metrics `*_admission_latencies_milliseconds` and `*_admission_latencies_milliseconds_summary` for backward compatible, but will be removed in a future release. ([#72343](https://github.com/kubernetes/kubernetes/pull/72343), [@danielqsj](https://github.com/danielqsj))",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72343",
-    "pr_number": 72343,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.0"
-  },
-  "72434": {
-    "commit": "07cf5fa08e84ce31c5deb2c43256d1f27bd3ba1a",
-    "text": "e2e storage tests run faster and are easier to read",
-    "markdown": "e2e storage tests run faster and are easier to read ([#72434](https://github.com/kubernetes/kubernetes/pull/72434), [@pohly](https://github.com/pohly))",
-    "author": "pohly",
-    "author_url": "https://github.com/pohly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72434",
-    "pr_number": 72434,
-    "kinds": ["cleanup"],
-    "sigs": ["testing"],
-    "release_version": "1.14.0"
-  },
-  "72470": {
-    "commit": "0b01b9ce51cf50284613bca4bdae3d486fcee56d",
-    "text": "Change kubelet metrics to conform metrics guidelines.\nThe following metrics are deprecated, and will be removed in a future release:\n- `kubelet_pod_worker_latency_microseconds`\n- `kubelet_pod_start_latency_microseconds`\n- `kubelet_cgroup_manager_latency_microseconds`\n- `kubelet_pod_worker_start_latency_microseconds`\n- `kubelet_pleg_relist_latency_microseconds`\n- `kubelet_pleg_relist_interval_microseconds`\n- `kubelet_eviction_stats_age_microseconds`\n- `kubelet_runtime_operations`\n- `kubelet_runtime_operations_latency_microseconds`\n- `kubelet_runtime_operations_errors`\n- `kubelet_device_plugin_registration_count`\n- `kubelet_device_plugin_alloc_latency_microseconds`\nPlease convert to the following metrics:\n- `kubelet_pod_worker_duration_seconds`\n- `kubelet_pod_start_duration_seconds`\n- `kubelet_cgroup_manager_duration_seconds`\n- `kubelet_pod_worker_start_duration_seconds`\n- `kubelet_pleg_relist_duration_seconds`\n- `kubelet_pleg_relist_interval_seconds`\n- `kubelet_eviction_stats_age_seconds`\n- `kubelet_runtime_operations_total`\n- `kubelet_runtime_operations_duration_seconds`\n- `kubelet_runtime_operations_errors_total`\n- `kubelet_device_plugin_registration_total`\n- `kubelet_device_plugin_alloc_duration_seconds`",
-    "markdown": "Change kubelet metrics to conform metrics guidelines.\n  The following metrics are deprecated, and will be removed in a future release:\n  - `kubelet_pod_worker_latency_microseconds`\n  - `kubelet_pod_start_latency_microseconds`\n  - `kubelet_cgroup_manager_latency_microseconds`\n  - `kubelet_pod_worker_start_latency_microseconds`\n  - `kubelet_pleg_relist_latency_microseconds`\n  - `kubelet_pleg_relist_interval_microseconds`\n  - `kubelet_eviction_stats_age_microseconds`\n  - `kubelet_runtime_operations`\n  - `kubelet_runtime_operations_latency_microseconds`\n  - `kubelet_runtime_operations_errors`\n  - `kubelet_device_plugin_registration_count`\n  - `kubelet_device_plugin_alloc_latency_microseconds`\n  Please convert to the following metrics:\n  - `kubelet_pod_worker_duration_seconds`\n  - `kubelet_pod_start_duration_seconds`\n  - `kubelet_cgroup_manager_duration_seconds`\n  - `kubelet_pod_worker_start_duration_seconds`\n  - `kubelet_pleg_relist_duration_seconds`\n  - `kubelet_pleg_relist_interval_seconds`\n  - `kubelet_eviction_stats_age_seconds`\n  - `kubelet_runtime_operations_total`\n  - `kubelet_runtime_operations_duration_seconds`\n  - `kubelet_runtime_operations_errors_total`\n  - `kubelet_device_plugin_registration_total`\n  - `kubelet_device_plugin_alloc_duration_seconds` ([#72470](https://github.com/kubernetes/kubernetes/pull/72470), [@danielqsj](https://github.com/danielqsj))\n\n  Courtesy of SIG Instrumentation, SIG Node, and SIG Testing",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72470",
-    "pr_number": 72470,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["instrumentation", "node", "testing"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "72491": {
-    "commit": "834c9a5e3d15c1114430fb4d5f1bc909c3235c68",
-    "text": "`system:kube-controller-manager` and `system:kube-scheduler` users are now permitted to perform delegated authentication/authorization checks by default RBAC policy",
-    "markdown": "`system:kube-controller-manager` and `system:kube-scheduler` users are now permitted to perform delegated authentication/authorization checks by default RBAC policy ([#72491](https://github.com/kubernetes/kubernetes/pull/72491), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72491",
-    "pr_number": 72491,
-    "kinds": ["bug"],
-    "sigs": ["auth", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "72507": {
-    "commit": "cb09d4d3179ba992f46b5efdb983fa04562f3d6c",
-    "text": "remove stale OutOfDisk condition from kubelet side",
-    "markdown": "remove stale OutOfDisk condition from kubelet side ([#72507](https://github.com/kubernetes/kubernetes/pull/72507), [@dixudx](https://github.com/dixudx))",
-    "author": "dixudx",
-    "author_url": "https://github.com/dixudx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72507",
-    "pr_number": 72507,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.14.0"
-  },
-  "72552": {
-    "commit": "50cf168e83f942f4850191c268461d07fcac19c7",
-    "text": "Promote ValidateProxyRedirects to Beta, and enable by default. This feature restricts redirect following from the apiserver to same-host redirects.\n\nIf nodes are configured to respond to CRI streaming requests on a different host interface than what the apiserver makes requests on (only the case if not using the built-in dockershim \u0026 setting the kubelet flag `--redirect-container-streaming=true`), then these requests will be broken. In that case, the feature can be temporarily disabled until the node configuration is corrected. We suggest setting `--redirect-container-streaming=false` on the kubelet to avoid issues.",
-    "markdown": "Promote ValidateProxyRedirects to Beta, and enable by default. This feature restricts redirect following from the apiserver to same-host redirects.\n  \n  If nodes are configured to respond to CRI streaming requests on a different host interface than what the apiserver makes requests on (only the case if not using the built-in dockershim \u0026 setting the kubelet flag `--redirect-container-streaming=true`), then these requests will be broken. In that case, the feature can be temporarily disabled until the node configuration is corrected. We suggest setting `--redirect-container-streaming=false` on the kubelet to avoid issues. ([#72552](https://github.com/kubernetes/kubernetes/pull/72552), [@tallclair](https://github.com/tallclair))\n\n  Courtesy of SIG API Machinery, and SIG Node",
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72552",
-    "pr_number": 72552,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "node"],
-    "feature": true,
-    "action_required": true,
-    "release_version": "1.14.0"
-  },
-  "72558": {
-    "commit": "d857790d360c2f9f86a47964017c08e409312551",
-    "text": "add goroutine to move unschedulable pods to activeq if they are not retried for more than 1 minute",
-    "markdown": "add goroutine to move unschedulable pods to activeq if they are not retried for more than 1 minute ([#72558](https://github.com/kubernetes/kubernetes/pull/72558), [@denkensk](https://github.com/denkensk))",
-    "author": "denkensk",
-    "author_url": "https://github.com/denkensk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72558",
-    "pr_number": 72558,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.0"
-  },
-  "72584": {
-    "commit": "6ce59dd1a25429663a801363dd57eed83d598408",
-    "text": "Install CSINodeInfo and CSIDriver CRDs in the local cluster.",
-    "markdown": "Install CSINodeInfo and CSIDriver CRDs in the local cluster. ([#72584](https://github.com/kubernetes/kubernetes/pull/72584), [@xing-yang](https://github.com/xing-yang))\n\n  Courtesy of SIG Storage",
-    "author": "xing-yang",
-    "author_url": "https://github.com/xing-yang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72584",
-    "pr_number": 72584,
-    "kinds": ["feature"],
-    "sigs": ["storage"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "72641": {
-    "commit": "9ecce4fdcf29a9352512ad4cfba7fe0e2192cd23",
-    "text": "kubectl supports copying files with wild card",
-    "markdown": "kubectl supports copying files with wild card ([#72641](https://github.com/kubernetes/kubernetes/pull/72641), [@dixudx](https://github.com/dixudx))\n\n  Courtesy of SIG CLI",
-    "author": "dixudx",
-    "author_url": "https://github.com/dixudx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72641",
-    "pr_number": 72641,
-    "areas": ["kubectl"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "72678": {
-    "commit": "fc1e528c4a53bdcd5e7f7cb61e58239d1aa21c96",
-    "text": "Add Custom Resource support to \"kubectl autoscale\"",
-    "markdown": "Add Custom Resource support to \"kubectl autoscale\" ([#72678](https://github.com/kubernetes/kubernetes/pull/72678), [@rmohr](https://github.com/rmohr))\n\n  Courtesy of SIG Autoscaling, and SIG CLI",
-    "author": "rmohr",
-    "author_url": "https://github.com/rmohr",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72678",
-    "pr_number": 72678,
-    "areas": ["custom-resources", "kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["autoscaling", "cli"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "72687": {
-    "commit": "72b0971f3b6a1ed96a571676c8f41726cd15612b",
-    "text": "This change applies zone labels to vSphere Volumes automatically. The zone labels are visible on the PV:\n$ kubectl get pv --show-labels\nNAME           CAPACITY   ACCESSMODES   STATUS    CLAIM            REASON    AGE       LABELS\npv-abc                5Gi        RWO                   Bound     default/claim1                    46s       failure-domain.beta.kubernetes.io/region=VC1,failure-domain.beta.kubernetes.io/zone=cluster-1",
-    "markdown": "This change applies zone labels to vSphere Volumes automatically. The zone labels are visible on the PV:\n  $ kubectl get pv --show-labels\n  NAME           CAPACITY   ACCESSMODES   STATUS    CLAIM            REASON    AGE       LABELS\n  pv-abc                5Gi        RWO                   Bound     default/claim1                    46s       failure-domain.beta.kubernetes.io/region=VC1,failure-domain.beta.kubernetes.io/zone=cluster-1 ([#72687](https://github.com/kubernetes/kubernetes/pull/72687), [@subramanian-neelakantan](https://github.com/subramanian-neelakantan))",
-    "author": "subramanian-neelakantan",
-    "author_url": "https://github.com/subramanian-neelakantan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72687",
-    "pr_number": 72687,
-    "areas": ["apiserver", "kubelet", "provider/vmware"],
-    "kinds": ["bug"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "auth",
-      "cloud-provider",
-      "network",
-      "node",
-      "scheduling",
-      "storage",
-      "testing"
-    ],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "72709": {
-    "commit": "289a60ad71669cd8fe376cdf4685ef807753fdd8",
-    "text": "when pleg channel is full, discard events and record its count",
-    "markdown": "when pleg channel is full, discard events and record its count ([#72709](https://github.com/kubernetes/kubernetes/pull/72709), [@changyaowei](https://github.com/changyaowei))",
-    "author": "changyaowei",
-    "author_url": "https://github.com/changyaowei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72709",
-    "pr_number": 72709,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.0"
-  },
-  "72729": {
-    "commit": "a99d6cef868338e613e80780d454accc839070e6",
-    "text": "The new test ``[sig-network] DNS should provide /etc/hosts entries for the cluster [LinuxOnly] [Conformance]`` will validate the host entries set in the ``/etc/hosts`` file (pod's FQDN and hostname), which should be managed by Kubelet.\n\nThe test has the tag ``[LinuxOnly]`` because individual files cannot be mounted in Windows Containers, which means that it cannot pass using Windows nodes.",
-    "markdown": "The new test ``[sig-network] DNS should provide /etc/hosts entries for the cluster [LinuxOnly] [Conformance]`` will validate the host entries set in the ``/etc/hosts`` file (pod's FQDN and hostname), which should be managed by Kubelet.\n  \n  The test has the tag ``[LinuxOnly]`` because individual files cannot be mounted in Windows Containers, which means that it cannot pass using Windows nodes. ([#72729](https://github.com/kubernetes/kubernetes/pull/72729), [@claudiubelu](https://github.com/claudiubelu))\n\n  Courtesy of SIG Architecture, SIG Network, SIG Testing, and SIG Windows",
-    "author": "claudiubelu",
-    "author_url": "https://github.com/claudiubelu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72729",
-    "pr_number": 72729,
-    "areas": ["conformance"],
-    "kinds": ["failing-test", "feature"],
-    "sigs": ["architecture", "network", "testing", "windows"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "72730": {
-    "commit": "44d1859205e22c6e86a3a1eecc599ac4279e9780",
-    "text": "Pod eviction now honors graceful deletion by default if no delete options are provided in the eviction request",
-    "markdown": "Pod eviction now honors graceful deletion by default if no delete options are provided in the eviction request ([#72730](https://github.com/kubernetes/kubernetes/pull/72730), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72730",
-    "pr_number": 72730,
-    "kinds": ["bug"],
-    "sigs": ["apps", "scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "72731": {
-    "commit": "701f914f3ac2d17230eb033e3313f8f8e89ed6a9",
-    "text": "This change ensures that volumes get provisioned based on the zone information provided in allowedTopologies.\n\nStorage class spec:\nkind: StorageClass\napiVersion: storage.k8s.io/v1\nmetadata:\n  name: fastpolicy1\nprovisioner: kubernetes.io/vsphere-volume\nparameters:\n    diskformat: zeroedthick\n    storagePolicyName: vSAN Default Storage Policy\nallowedTopologies:\n- matchLabelExpressions:\n  - key: failure-domain.beta.kubernetes.io/zone\n    values:\n    - zone1\n\nPV creation Logs:\nI0109 11:17:52.321372       1 vsphere.go:1147] Starting to create a vSphere volume with volumeOptions: \u0026{CapacityKB:1048576 Tags:map[kubernetes.io/created-for/pvc/namespace:default kubernetes.io/created-for/pvc/name:pvcsc-1-policy kubernetes.io/created-for/pv/name:pvc-34650c12-1400-11e9-aef4-005056804cc9] Name:kubernetes-dynamic-pvc-34650c12-1400-11e9-aef4-005056804cc9 DiskFormat:zeroedthick Datastore: VSANStorageProfileData: StoragePolicyName:vSAN Default Storage Policy StoragePolicyID: SCSIControllerType: Zone:[zone1]}\n...\nI0109 11:17:59.430113       1 vsphere.go:1334] The canonical volume path for the newly created vSphere volume is \"[vsanDatastore] 98db185c-6683-d8c7-bc55-0200435ec5da/kubernetes-dynamic-pvc-34650c12-1400-11e9-aef4-005056804cc9.vmdk\"\n\nRan regression tests (no zone) and they passed.",
-    "markdown": "This change ensures that volumes get provisioned based on the zone information provided in allowedTopologies.\n  \n  Storage class spec:\n  kind: StorageClass\n  apiVersion: storage.k8s.io/v1\n  metadata:\n    name: fastpolicy1\n  provisioner: kubernetes.io/vsphere-volume\n  parameters:\n      diskformat: zeroedthick\n      storagePolicyName: vSAN Default Storage Policy\n  allowedTopologies:\n  - matchLabelExpressions:\n    - key: failure-domain.beta.kubernetes.io/zone\n      values:\n      - zone1\n  \n  PV creation Logs:\n  I0109 11:17:52.321372       1 vsphere.go:1147] Starting to create a vSphere volume with volumeOptions: \u0026{CapacityKB:1048576 Tags:map[kubernetes.io/created-for/pvc/namespace:default kubernetes.io/created-for/pvc/name:pvcsc-1-policy kubernetes.io/created-for/pv/name:pvc-34650c12-1400-11e9-aef4-005056804cc9] Name:kubernetes-dynamic-pvc-34650c12-1400-11e9-aef4-005056804cc9 DiskFormat:zeroedthick Datastore: VSANStorageProfileData: StoragePolicyName:vSAN Default Storage Policy StoragePolicyID: SCSIControllerType: Zone:[zone1]}\n  ...\n  I0109 11:17:59.430113       1 vsphere.go:1334] The canonical volume path for the newly created vSphere volume is \"[vsanDatastore] 98db185c-6683-d8c7-bc55-0200435ec5da/kubernetes-dynamic-pvc-34650c12-1400-11e9-aef4-005056804cc9.vmdk\"\n  \n  Ran regression tests (no zone) and they passed. ([#72731](https://github.com/kubernetes/kubernetes/pull/72731), [@skarthiksrinivas](https://github.com/skarthiksrinivas))",
-    "author": "skarthiksrinivas",
-    "author_url": "https://github.com/skarthiksrinivas",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72731",
-    "pr_number": 72731,
-    "areas": ["provider/vmware"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "72785": {
-    "commit": "dc1244c6cde46760fb91b0e552fdff98397bc9d3",
-    "text": "The HugePages feature gate has graduated to GA, and can no longer be disabled. The feature gate will be removed in v1.16",
-    "markdown": "The HugePages feature gate has graduated to GA, and can no longer be disabled. The feature gate will be removed in v1.16 ([#72785](https://github.com/kubernetes/kubernetes/pull/72785), [@derekwaynecarr](https://github.com/derekwaynecarr))\n\n  Courtesy of SIG Apps, and SIG Node",
-    "author": "derekwaynecarr",
-    "author_url": "https://github.com/derekwaynecarr",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72785",
-    "pr_number": 72785,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["apps", "node"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "72832": {
-    "commit": "235b32e8ad978d0ef0b801baacb5bdecd6e2d5f7",
-    "text": "The CustomPodDNS feature gate has graduated to GA, and can no longer be disabled. The feature gate will be removed in v1.16",
-    "markdown": "The CustomPodDNS feature gate has graduated to GA, and can no longer be disabled. The feature gate will be removed in v1.16 ([#72832](https://github.com/kubernetes/kubernetes/pull/72832), [@MrHohn](https://github.com/MrHohn))\n\n  Courtesy of SIG Apps, SIG Network, and SIG Node",
-    "author": "MrHohn",
-    "author_url": "https://github.com/MrHohn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72832",
-    "pr_number": 72832,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["apps", "network", "node"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "72836": {
-    "commit": "8178fa1b13b54a23bf45da9e84031aea357df4fc",
-    "text": "the test/e2e/e2e.test binary can test arbitrary storage drivers, see the `-storage.testdriver` parameter",
-    "markdown": "the test/e2e/e2e.test binary can test arbitrary storage drivers, see the `-storage.testdriver` parameter ([#72836](https://github.com/kubernetes/kubernetes/pull/72836), [@pohly](https://github.com/pohly))\n\n  Courtesy of SIG Storage, and SIG Testing",
-    "author": "pohly",
-    "author_url": "https://github.com/pohly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72836",
-    "pr_number": 72836,
-    "kinds": ["feature"],
-    "sigs": ["storage", "testing"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "72844": {
-    "commit": "5d19fda5e831ad09b8324825a5b66c4163da120a",
-    "text": "PidPressure evicts pods from lowest priority to highest priority",
-    "markdown": "PidPressure evicts pods from lowest priority to highest priority ([#72844](https://github.com/kubernetes/kubernetes/pull/72844), [@dashpole](https://github.com/dashpole))",
-    "author": "dashpole",
-    "author_url": "https://github.com/dashpole",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72844",
-    "pr_number": 72844,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "72870": {
-    "commit": "6d479af411333c8f1bba2c1edac76ec929caf1f7",
-    "text": "kubeadm: pull images when joining a new control plane instance",
-    "markdown": "kubeadm: pull images when joining a new control plane instance ([#72870](https://github.com/kubernetes/kubernetes/pull/72870), [@MalloZup](https://github.com/MalloZup))",
-    "author": "MalloZup",
-    "author_url": "https://github.com/MalloZup",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72870",
-    "pr_number": 72870,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "72910": {
-    "commit": "a20cd49d6db082ff350c73a424250b0ee64a854a",
-    "text": "Add kubelet_node_name metrics.",
-    "markdown": "Add kubelet_node_name metrics. ([#72910](https://github.com/kubernetes/kubernetes/pull/72910), [@danielqsj](https://github.com/danielqsj))\n\n  Courtesy of SIG Instrumentation, and SIG Node",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72910",
-    "pr_number": 72910,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["instrumentation", "node"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "72924": {
-    "commit": "7a8ac587564e5e7e42d664322b64328e7e948788",
-    "text": "The `/swaggerapi/*` schema docs, deprecated since 1.7, have been removed in favor of the /openapi/v2 schema docs.",
-    "markdown": "The `/swaggerapi/*` schema docs, deprecated since 1.7, have been removed in favor of the /openapi/v2 schema docs. ([#72924](https://github.com/kubernetes/kubernetes/pull/72924), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72924",
-    "pr_number": 72924,
-    "areas": ["apiserver", "kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "auth", "cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "72970": {
-    "commit": "26d58d37a71ecc698259729361484007b45a82ba",
-    "text": "Fix graceful apiserver shutdown to not drop outgoing bytes before the process terminates.",
-    "markdown": "Fix graceful apiserver shutdown to not drop outgoing bytes before the process terminates. ([#72970](https://github.com/kubernetes/kubernetes/pull/72970), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72970",
-    "pr_number": 72970,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cloud-provider", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "72978": {
-    "commit": "d306d68a36717167d8aec99b9684ee0867089ab9",
-    "text": "kube-addon-manager was updated to v9.0, and now uses kubectl v1.13.2 and prunes workload resources via the apps/v1 API",
-    "markdown": "kube-addon-manager was updated to v9.0, and now uses kubectl v1.13.2 and prunes workload resources via the apps/v1 API ([#72978](https://github.com/kubernetes/kubernetes/pull/72978), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72978",
-    "pr_number": 72978,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "72980": {
-    "commit": "d6b7409103772be8dcaf40d8c2a753c4231404d8",
-    "text": "Add support for max attach limit for Cinder",
-    "markdown": "Add support for max attach limit for Cinder ([#72980](https://github.com/kubernetes/kubernetes/pull/72980), [@gnufied](https://github.com/gnufied))\n\n  Courtesy of SIG Scheduling, and SIG Storage",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72980",
-    "pr_number": 72980,
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "storage"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "72984": {
-    "commit": "f2b133d7b46199a16572a9f59dedea8e45958051",
-    "text": "kubeadm: explicitly wait for `etcd` to have grown when joining a new control plane",
-    "markdown": "kubeadm: explicitly wait for `etcd` to have grown when joining a new control plane ([#72984](https://github.com/kubernetes/kubernetes/pull/72984), [@ereslibre](https://github.com/ereslibre))",
-    "author": "ereslibre",
-    "author_url": "https://github.com/ereslibre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72984",
-    "pr_number": 72984,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "73001": {
-    "commit": "4cd759dbe0a5239de01ea8ece2d4f73a5dd8706c",
-    "text": "As per deprecation policy in https://kubernetes.io/docs/reference/using-api/deprecation-policy/ \nthe taints \"node.alpha.kubernetes.io/notReady\" and \"node.alpha.kubernetes.io/unreachable\".  are no\nlonger supported or adjusted. These uses should be replaced with \"node.kubernetes.io/not-ready\" \nand \"node.kubernetes.io/unreachable\" respectively instead.",
-    "markdown": "As per deprecation policy in https://kubernetes.io/docs/reference/using-api/deprecation-policy/ \n  the taints \"node.alpha.kubernetes.io/notReady\" and \"node.alpha.kubernetes.io/unreachable\".  are no\n  longer supported or adjusted. These uses should be replaced with \"node.kubernetes.io/not-ready\" \n  and \"node.kubernetes.io/unreachable\" respectively instead. ([#73001](https://github.com/kubernetes/kubernetes/pull/73001), [@shivnagarajan](https://github.com/shivnagarajan))",
-    "author": "shivnagarajan",
-    "author_url": "https://github.com/shivnagarajan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73001",
-    "pr_number": 73001,
-    "kinds": ["cleanup"],
-    "sigs": ["apps", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "73007": {
-    "commit": "8c95a07b4bf179fd64a84215e041401a313ce0f6",
-    "text": "The `RunAsGroup` feature has been promoted to beta and enabled by default. PodSpec and PodSecurityPolicy objects can be used to control the primary GID of containers on supported container runtimes.",
-    "markdown": "The `RunAsGroup` feature has been promoted to beta and enabled by default. PodSpec and PodSecurityPolicy objects can be used to control the primary GID of containers on supported container runtimes. ([#73007](https://github.com/kubernetes/kubernetes/pull/73007), [@krmayankk](https://github.com/krmayankk))\n\n  Courtesy of SIG Auth, SIG Node, and SIG Testing",
-    "author": "krmayankk",
-    "author_url": "https://github.com/krmayankk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73007",
-    "pr_number": 73007,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["auth", "node", "testing"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "73022": {
-    "commit": "c999dc3854d7b14aa28b7c61622a8bc2975e5367",
-    "text": "acquire lock before operating unschedulablepodsmap",
-    "markdown": "acquire lock before operating unschedulablepodsmap ([#73022](https://github.com/kubernetes/kubernetes/pull/73022), [@denkensk](https://github.com/denkensk))",
-    "author": "denkensk",
-    "author_url": "https://github.com/denkensk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73022",
-    "pr_number": 73022,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.0"
-  },
-  "73033": {
-    "commit": "66149bd61d895e458164a768e2a388dc0da805e8",
-    "text": "Kustomize is developed in its own repo https://github.com/kubernetes-sigs/kustomize\n\nThis PR added a new subcommand `kustomize` in kubectl. \n  kubectl kustomize \u003csomedir\u003e has the same effect as kustomize build \u003csomedir\u003e\n\nTo build API resources from somedir with a kustomization.yaml file\n   kubectl kustomize \u003csomedir\u003e\n\nThis command can be piped to apply or delete \n   kubectl kustomize \u003csomedir\u003e | kubectl apply -f -\n   kubectl kustomize \u003csomedir\u003e | kubectl delete -f -",
-    "markdown": "Kustomize is developed in its own repo https://github.com/kubernetes-sigs/kustomize\n  \n  This PR added a new subcommand `kustomize` in kubectl. \n    kubectl kustomize \u003csomedir\u003e has the same effect as kustomize build \u003csomedir\u003e\n  \n  To build API resources from somedir with a kustomization.yaml file\n     kubectl kustomize \u003csomedir\u003e\n  \n  This command can be piped to apply or delete \n     kubectl kustomize \u003csomedir\u003e | kubectl apply -f -\n     kubectl kustomize \u003csomedir\u003e | kubectl delete -f - ([#73033](https://github.com/kubernetes/kubernetes/pull/73033), [@Liujingfang1](https://github.com/Liujingfang1))\n\n  Courtesy of SIG CLI",
-    "author": "Liujingfang1",
-    "author_url": "https://github.com/Liujingfang1",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73033",
-    "pr_number": 73033,
-    "areas": ["kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "73035": {
-    "commit": "0a7d8ec6cc40099e40567560a9d101e0cc69a6d8",
-    "text": "The \"kubectl api-resources\" command will no longer fail to display any resources on a single failure",
-    "markdown": "The \"kubectl api-resources\" command will no longer fail to display any resources on a single failure ([#73035](https://github.com/kubernetes/kubernetes/pull/73035), [@juanvallejo](https://github.com/juanvallejo))",
-    "author": "juanvallejo",
-    "author_url": "https://github.com/juanvallejo",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73035",
-    "pr_number": 73035,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.14.0"
-  },
-  "73048": {
-    "commit": "76961c517e538206dd121c60a37736df9c328726",
-    "text": "Node OS/arch labels are promoted to GA",
-    "markdown": "Node OS/arch labels are promoted to GA ([#73048](https://github.com/kubernetes/kubernetes/pull/73048), [@yujuhong](https://github.com/yujuhong))\n\n  Courtesy of SIG Node",
-    "author": "yujuhong",
-    "author_url": "https://github.com/yujuhong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73048",
-    "pr_number": 73048,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "73063": {
-    "commit": "29e9ff36a93ab2829f08550a42008126a11a61f5",
-    "text": "Now users could get object info like:\na. kubectl get pod test-pod -o custom-columns=CONTAINER:.spec.containers[0:3].name\nb. kubectl get pod test-pod -o custom-columns=CONTAINER:.spec.containers[-2:].name",
-    "markdown": "Now users could get object info like:\n  a. kubectl get pod test-pod -o custom-columns=CONTAINER:.spec.containers[0:3].name\n  b. kubectl get pod test-pod -o custom-columns=CONTAINER:.spec.containers[-2:].name ([#73063](https://github.com/kubernetes/kubernetes/pull/73063), [@WanLinghao](https://github.com/WanLinghao))",
-    "author": "WanLinghao",
-    "author_url": "https://github.com/WanLinghao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73063",
-    "pr_number": 73063,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.14.0"
-  },
-  "73078": {
-    "commit": "0b8566f3887a2d13baba623d88aeb64f6e637f46",
-    "text": "scheduler: makes pod less racing so as to be put back into activeQ properly",
-    "markdown": "scheduler: makes pod less racing so as to be put back into activeQ properly ([#73078](https://github.com/kubernetes/kubernetes/pull/73078), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73078",
-    "pr_number": 73078,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.0"
-  },
-  "73093": {
-    "commit": "ce52bf1265bfd91158f980ae965388bb097d97e2",
-    "text": "kubeadm: When certificates are present joining a new control plane make sure that they match at least the required SANs",
-    "markdown": "kubeadm: When certificates are present joining a new control plane make sure that they match at least the required SANs ([#73093](https://github.com/kubernetes/kubernetes/pull/73093), [@ereslibre](https://github.com/ereslibre))",
-    "author": "ereslibre",
-    "author_url": "https://github.com/ereslibre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73093",
-    "pr_number": 73093,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "73097": {
-    "commit": "d654b49c0eb3b203dc704d9772e3d0751d4734d3",
-    "text": "A new `TaintNodesByCondition` admission plugin taints newly created Node objects as \"not ready\", to fix a race condition that could cause pods to be scheduled on new nodes before their taints were updated to accurately reflect their reported conditions. This admission plugin is enabled by default if the `TaintNodesByCondition` feature is enabled.",
-    "markdown": "A new `TaintNodesByCondition` admission plugin taints newly created Node objects as \"not ready\", to fix a race condition that could cause pods to be scheduled on new nodes before their taints were updated to accurately reflect their reported conditions. This admission plugin is enabled by default if the `TaintNodesByCondition` feature is enabled. ([#73097](https://github.com/kubernetes/kubernetes/pull/73097), [@bsalamat](https://github.com/bsalamat))",
-    "author": "bsalamat",
-    "author_url": "https://github.com/bsalamat",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73097",
-    "pr_number": 73097,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth", "node", "scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "73102": {
-    "commit": "5fc286fb3ca5600b038c67c9b5e66577b2110c43",
-    "text": "If you are running the cloud-controller-manager and you have the `pvlabel.kubernetes.io` alpha Initializer enabled, you must now enable PersistentVolume labeling using the `PersistentVolumeLabel` admission controller instead. You can do this by adding `PersistentVolumeLabel` in the `--enable-admission-plugins` kube-apiserver flag.",
-    "markdown": "If you are running the cloud-controller-manager and you have the `pvlabel.kubernetes.io` alpha Initializer enabled, you must now enable PersistentVolume labeling using the `PersistentVolumeLabel` admission controller instead. You can do this by adding `PersistentVolumeLabel` in the `--enable-admission-plugins` kube-apiserver flag. ([#73102](https://github.com/kubernetes/kubernetes/pull/73102), [@andrewsykim](https://github.com/andrewsykim))\n\n  Courtesy of SIG Cloud Provider, and SIG Storage",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73102",
-    "pr_number": 73102,
-    "areas": ["provider/openstack"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider", "storage"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "73115": {
-    "commit": "cbf52183fa3b01b877fad0ecf45ba46b4d9d5d72",
-    "text": "Fixed a bug that caused PV allocation on non-English vSphere installations to fail",
-    "markdown": "Fixed a bug that caused PV allocation on non-English vSphere installations to fail ([#73115](https://github.com/kubernetes/kubernetes/pull/73115), [@alvaroaleman](https://github.com/alvaroaleman))",
-    "author": "alvaroaleman",
-    "author_url": "https://github.com/alvaroaleman",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73115",
-    "pr_number": 73115,
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.0"
-  },
-  "73138": {
-    "commit": "56172863598bf435cd08f2ee0d77e9b635975dba",
-    "text": "metadata.deletionTimestamp is no longer moved into the future when issuing repeated DELETE requests against a resource containing a finalizer.",
-    "markdown": "metadata.deletionTimestamp is no longer moved into the future when issuing repeated DELETE requests against a resource containing a finalizer. ([#73138](https://github.com/kubernetes/kubernetes/pull/73138), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73138",
-    "pr_number": 73138,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.0"
-  },
-  "73148": {
-    "commit": "31311a634345fea7f6be68cb73bed7481cc127c7",
-    "text": "The /swagger.json and /swagger-2.0.0.pb-v1 schema documents, deprecated since v1.10, have been removed in favor of `/openapi/v2`",
-    "markdown": "The /swagger.json and /swagger-2.0.0.pb-v1 schema documents, deprecated since v1.10, have been removed in favor of `/openapi/v2` ([#73148](https://github.com/kubernetes/kubernetes/pull/73148), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73148",
-    "pr_number": 73148,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "auth", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "73149": {
-    "commit": "27e5971c11cfcda703a39ed670a565f0f3564713",
-    "text": "jsonpath expressions containing `[start:end:step]` slice are now evaluated correctly",
-    "markdown": "jsonpath expressions containing `[start:end:step]` slice are now evaluated correctly ([#73149](https://github.com/kubernetes/kubernetes/pull/73149), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73149",
-    "pr_number": 73149,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.0"
-  },
-  "73191": {
-    "commit": "81af41836b347cdb2ee9138c728cc7516d3dd126",
-    "text": "Added alpha field storageVersionHash to the discovery document for each resource. Its value must be treated as opaque by clients. Only equality comparison on the value is valid.",
-    "markdown": "Added alpha field storageVersionHash to the discovery document for each resource. Its value must be treated as opaque by clients. Only equality comparison on the value is valid. ([#73191](https://github.com/kubernetes/kubernetes/pull/73191), [@caesarxuchao](https://github.com/caesarxuchao))\n\n  Courtesy of SIG API Machinery",
-    "author": "caesarxuchao",
-    "author_url": "https://github.com/caesarxuchao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73191",
-    "pr_number": 73191,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "73239": {
-    "commit": "20a11ac5ccd8fc5644eccf98e42db9a642bbed21",
-    "text": "kubeadm: add back `--cert-dir` option for `kubeadm init phase certs sa`",
-    "markdown": "kubeadm: add back `--cert-dir` option for `kubeadm init phase certs sa` ([#73239](https://github.com/kubernetes/kubernetes/pull/73239), [@mattkelly](https://github.com/mattkelly))",
-    "author": "mattkelly",
-    "author_url": "https://github.com/mattkelly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73239",
-    "pr_number": 73239,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "73250": {
-    "commit": "46b2255e97580ca5908770affb768ab2ed4dbf65",
-    "text": "The `rules` field in RBAC Role and ClusterRole objects is now correctly reported as optional in the openapi schema.",
-    "markdown": "The `rules` field in RBAC Role and ClusterRole objects is now correctly reported as optional in the openapi schema. ([#73250](https://github.com/kubernetes/kubernetes/pull/73250), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73250",
-    "pr_number": 73250,
-    "kinds": ["api-change", "bug"],
-    "sigs": ["architecture", "auth"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "73256": {
-    "commit": "73dc138cae7c27e4d50d4dc654800891d06bb5b6",
-    "text": "Update kubelet CLI summary documentation and generated Webpage",
-    "markdown": "Update kubelet CLI summary documentation and generated Webpage ([#73256](https://github.com/kubernetes/kubernetes/pull/73256), [@deitch](https://github.com/deitch))",
-    "author": "deitch",
-    "author_url": "https://github.com/deitch",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73256",
-    "pr_number": 73256,
-    "areas": ["kubelet"],
-    "kinds": ["documentation"],
-    "sigs": ["node"],
-    "release_version": "1.14.0"
-  },
-  "73266": {
-    "commit": "e9a8d27bd49955fadfd87c748b8857d1d2a3d60b",
-    "text": "The `--quiet` option to `kubectl run` now suppresses resource deletion messages emitted when the `--rm` option is specified.",
-    "markdown": "The `--quiet` option to `kubectl run` now suppresses resource deletion messages emitted when the `--rm` option is specified. ([#73266](https://github.com/kubernetes/kubernetes/pull/73266), [@awh](https://github.com/awh))\n\n  Courtesy of SIG CLI",
-    "author": "awh",
-    "author_url": "https://github.com/awh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73266",
-    "pr_number": 73266,
-    "areas": ["kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "73267": {
-    "commit": "c501d464afe91ccdf17232f8d911924017bf6970",
-    "text": "The CoreDNS configuration now has the forward plugin for proxy in the default configuration instead of the proxy plugin.",
-    "markdown": "The CoreDNS configuration now has the forward plugin for proxy in the default configuration instead of the proxy plugin. ([#73267](https://github.com/kubernetes/kubernetes/pull/73267), [@rajansandeep](https://github.com/rajansandeep))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "rajansandeep",
-    "author_url": "https://github.com/rajansandeep",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73267",
-    "pr_number": 73267,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "73268": {
-    "commit": "c21f60f8620da0cb1244839277bf16273decda3a",
-    "text": "Scale max-inflight limits together with master VM sizes.",
-    "markdown": "Scale max-inflight limits together with master VM sizes. ([#73268](https://github.com/kubernetes/kubernetes/pull/73268), [@wojtek-t](https://github.com/wojtek-t))",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73268",
-    "pr_number": 73268,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "scalability"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "73277": {
-    "commit": "4a44df3fefbf6e548f2d6875a6de79884befb09a",
-    "text": "When a watch is closed by an HTTP2 load balancer and we are told to go away, skip printing the message to stderr by default.",
-    "markdown": "When a watch is closed by an HTTP2 load balancer and we are told to go away, skip printing the message to stderr by default. ([#73277](https://github.com/kubernetes/kubernetes/pull/73277), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73277",
-    "pr_number": 73277,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.0"
-  },
-  "73288": {
-    "commit": "02b8056efbbc524abf157c58cfc38bd5fc69cf61",
-    "text": "Node-Problem-Detector configuration is now decoupled from the Kubernetes release on GKE/GCE.",
-    "markdown": "Node-Problem-Detector configuration is now decoupled from the Kubernetes release on GKE/GCE. ([#73288](https://github.com/kubernetes/kubernetes/pull/73288), [@wangzhen127](https://github.com/wangzhen127))",
-    "author": "wangzhen127",
-    "author_url": "https://github.com/wangzhen127",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73288",
-    "pr_number": 73288,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "node", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "73292": {
-    "commit": "296985ce35c203a882d4ea394a5172a65c7c34c6",
-    "text": "windows: Ensure graceful termination when being run as windows service",
-    "markdown": "windows: Ensure graceful termination when being run as windows service ([#73292](https://github.com/kubernetes/kubernetes/pull/73292), [@steffengy](https://github.com/steffengy))",
-    "author": "steffengy",
-    "author_url": "https://github.com/steffengy",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73292",
-    "pr_number": 73292,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "73309": {
-    "commit": "9388a27212a52f93f5f466ad8bcef513708ceb9e",
-    "text": "scheduler: use incremental scheduling cycle in PriorityQueue to put all in-flight unschedulable pods back to active queue if we received move request",
-    "markdown": "scheduler: use incremental scheduling cycle in PriorityQueue to put all in-flight unschedulable pods back to active queue if we received move request ([#73309](https://github.com/kubernetes/kubernetes/pull/73309), [@cofyc](https://github.com/cofyc))",
-    "author": "cofyc",
-    "author_url": "https://github.com/cofyc",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73309",
-    "pr_number": 73309,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.0"
-  },
-  "73315": {
-    "commit": "95e5705faf53dcd7bd19c44bf0024b2f1d9c24ff",
-    "text": "The timeout error message on wait command will contain resource name on which condition wasn't met.",
-    "markdown": "The timeout error message on wait command will contain resource name on which condition wasn't met. ([#73315](https://github.com/kubernetes/kubernetes/pull/73315), [@Zyqsempai](https://github.com/Zyqsempai))",
-    "author": "Zyqsempai",
-    "author_url": "https://github.com/Zyqsempai",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73315",
-    "pr_number": 73315,
-    "areas": ["kubectl"],
-    "sigs": ["cli"],
-    "release_version": "1.14.0"
-  },
-  "73326": {
-    "commit": "54015767a5dfef0c2332bb42daf58f4fd94527bc",
-    "text": "Update to go1.11.5",
-    "markdown": "Update to go1.11.5 ([#73326](https://github.com/kubernetes/kubernetes/pull/73326), [@ixdy](https://github.com/ixdy))",
-    "author": "ixdy",
-    "author_url": "https://github.com/ixdy",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73326",
-    "pr_number": 73326,
-    "areas": ["security"],
-    "kinds": ["bug"],
-    "sigs": ["release", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "73333": {
-    "commit": "3e748958dcd60de1e56339e957090111c410fb6e",
-    "text": "kubelet: OS and Arch information is now recorded in `kubernetes.io/os` and `kubernetes.io/arch` labels on Node objects. The previous labels (`beta.kubernetes.io/os` and `beta.kubernetes.io/arch`) are still recorded, but are deprecated and targeted for removal in 1.18.",
-    "markdown": "kubelet: OS and Arch information is now recorded in `kubernetes.io/os` and `kubernetes.io/arch` labels on Node objects. The previous labels (`beta.kubernetes.io/os` and `beta.kubernetes.io/arch`) are still recorded, but are deprecated and targeted for removal in 1.18. ([#73333](https://github.com/kubernetes/kubernetes/pull/73333), [@yujuhong](https://github.com/yujuhong))",
-    "author": "yujuhong",
-    "author_url": "https://github.com/yujuhong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73333",
-    "pr_number": 73333,
-    "areas": ["kubelet"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["architecture", "node"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "73342": {
-    "commit": "c9964aa519ffbd4904c8f49a46c753dffd48da66",
-    "text": "fix \u0026#35;73264 cpuPeriod was not reset, but used as set via flag, although it was disabled via alpha gate",
-    "markdown": "fix \u0026#35;73264 cpuPeriod was not reset, but used as set via flag, although it was disabled via alpha gate ([#73342](https://github.com/kubernetes/kubernetes/pull/73342), [@szuecs](https://github.com/szuecs))",
-    "author": "szuecs",
-    "author_url": "https://github.com/szuecs",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73342",
-    "pr_number": 73342,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.0"
-  },
-  "73345": {
-    "commit": "64ce2e598fec69e69352c4d5f203c1f8e341c17f",
-    "text": "Speedup kubectl by \u003e10 when calling out to kube-apiserver for discovery information.",
-    "markdown": "Speedup kubectl by \u003e10 when calling out to kube-apiserver for discovery information. ([#73345](https://github.com/kubernetes/kubernetes/pull/73345), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73345",
-    "pr_number": 73345,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "apps", "cli"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "73401": {
-    "commit": "f78770565ca84dd719afb5231fd09838b32c8fd3",
-    "text": "Adds deleting pods created by DaemonSet assigned to not existing nodes.",
-    "markdown": "Adds deleting pods created by DaemonSet assigned to not existing nodes. ([#73401](https://github.com/kubernetes/kubernetes/pull/73401), [@krzysztof-jastrzebski](https://github.com/krzysztof-jastrzebski))",
-    "author": "krzysztof-jastrzebski",
-    "author_url": "https://github.com/krzysztof-jastrzebski",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73401",
-    "pr_number": 73401,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.14.0"
-  },
-  "73402": {
-    "commit": "732cb10019c50e061b4a0ed652d1f116d3cc3e7e",
-    "text": "e2e.test now rejects unknown --provider values instead of merely warning about them. An empty provider name is not accepted anymore and was replaced by \"skeleton\" (= a provider with no special behavior).",
-    "markdown": "e2e.test now rejects unknown --provider values instead of merely warning about them. An empty provider name is not accepted anymore and was replaced by \"skeleton\" (= a provider with no special behavior). ([#73402](https://github.com/kubernetes/kubernetes/pull/73402), [@pohly](https://github.com/pohly))",
-    "author": "pohly",
-    "author_url": "https://github.com/pohly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73402",
-    "pr_number": 73402,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["testing"],
-    "release_version": "1.14.0"
-  },
-  "73435": {
-    "commit": "10a4d5aa8a3e0c6c9a347cece2b74d8054a28e51",
-    "text": "Updated AWS SDK to v1.16.26 for ECR PrivateLink support",
-    "markdown": "Updated AWS SDK to v1.16.26 for ECR PrivateLink support ([#73435](https://github.com/kubernetes/kubernetes/pull/73435), [@micahhausler](https://github.com/micahhausler))",
-    "author": "micahhausler",
-    "author_url": "https://github.com/micahhausler",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73435",
-    "pr_number": 73435,
-    "kinds": ["feature"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "73441": {
-    "commit": "31c6a2ba355687363ee694611148b7c67a0ae8d4",
-    "text": "kube-apiserver now only aggregates openapi schemas from `/openapi/v2` endpoints of aggregated API servers. The fallback to aggregate from `/swagger.json` has been removed. Ensure aggregated API servers provide schema information via `/openapi/v2` (available since v1.10).",
-    "markdown": "kube-apiserver now only aggregates openapi schemas from `/openapi/v2` endpoints of aggregated API servers. The fallback to aggregate from `/swagger.json` has been removed. Ensure aggregated API servers provide schema information via `/openapi/v2` (available since v1.10). ([#73441](https://github.com/kubernetes/kubernetes/pull/73441), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73441",
-    "pr_number": 73441,
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.0"
-  },
-  "73442": {
-    "commit": "244795a06aeeabf243b998aaaa6c6352eb3c9a62",
-    "text": "Introduced support for Windows nodes into the cluster bringup scripts for GCE.",
-    "markdown": "Introduced support for Windows nodes into the cluster bringup scripts for GCE. ([#73442](https://github.com/kubernetes/kubernetes/pull/73442), [@pjh](https://github.com/pjh))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "pjh",
-    "author_url": "https://github.com/pjh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73442",
-    "pr_number": 73442,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "73482": {
-    "commit": "fb96afb1946c3d94cd17123e1c7b2adec69cf0f3",
-    "text": "fixes an error processing watch events when running skewed apiservers",
-    "markdown": "fixes an error processing watch events when running skewed apiservers ([#73482](https://github.com/kubernetes/kubernetes/pull/73482), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73482",
-    "pr_number": 73482,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "73484": {
-    "commit": "7693a1d5fe2a35b6e2e205f03ae9b3eddcdabc6b",
-    "text": "Fix a bug that aggregated openapi spec may override swagger securityDefinitions and swagger info in kube-apiserver",
-    "markdown": "Fix a bug that aggregated openapi spec may override swagger securityDefinitions and swagger info in kube-apiserver ([#73484](https://github.com/kubernetes/kubernetes/pull/73484), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73484",
-    "pr_number": 73484,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.0"
-  },
-  "73498": {
-    "commit": "3b40af7d80fcf20f8fc6739051e284db6873a560",
-    "text": "Graduate Pod Priority and Preemption to GA.",
-    "markdown": "Graduate Pod Priority and Preemption to GA. ([#73498](https://github.com/kubernetes/kubernetes/pull/73498), [@bsalamat](https://github.com/bsalamat))\n\n  Courtesy of SIG Scheduling",
-    "author": "bsalamat",
-    "author_url": "https://github.com/bsalamat",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73498",
-    "pr_number": 73498,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "73533": {
-    "commit": "6fc9a72f5888a0dda1febbda468fdf09be4f4ca7",
-    "text": "Ensure directories on volumes are group-executable when using fsGroup",
-    "markdown": "Ensure directories on volumes are group-executable when using fsGroup ([#73533](https://github.com/kubernetes/kubernetes/pull/73533), [@mxey](https://github.com/mxey))",
-    "author": "mxey",
-    "author_url": "https://github.com/mxey",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73533",
-    "pr_number": 73533,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.0"
-  },
-  "73542": {
-    "commit": "ad58fb1ea09aca034bab83e57b396149ec4789c5",
-    "text": "Missing directories listed in a user's PATH are no longer considered errors and are instead logged by the \"kubectl plugin list\" command when listing available plugins.",
-    "markdown": "Missing directories listed in a user's PATH are no longer considered errors and are instead logged by the \"kubectl plugin list\" command when listing available plugins. ([#73542](https://github.com/kubernetes/kubernetes/pull/73542), [@juanvallejo](https://github.com/juanvallejo))",
-    "author": "juanvallejo",
-    "author_url": "https://github.com/juanvallejo",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73542",
-    "pr_number": 73542,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.14.0"
-  },
-  "73555": {
-    "commit": "3afa003126ff50092954839efbe10d584c2511ff",
-    "text": "The PriorityClass API has been promoted to `scheduling.k8s.io/v1` with no changes. The `scheduling.k8s.io/v1beta1` version is now deprecated and will stop being served by default in v1.17.",
-    "markdown": "The PriorityClass API has been promoted to `scheduling.k8s.io/v1` with no changes. The `scheduling.k8s.io/v1beta1` version is now deprecated and will stop being served by default in v1.17. ([#73555](https://github.com/kubernetes/kubernetes/pull/73555), [@bsalamat](https://github.com/bsalamat))\n\n  Courtesy of SIG API Machinery, SIG Architecture, SIG CLI, SIG Cloud Provider, SIG Scheduling, and SIG Testing",
-    "author": "bsalamat",
-    "author_url": "https://github.com/bsalamat",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73555",
-    "pr_number": 73555,
-    "areas": ["apiserver", "kubectl"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "architecture", "cli", "cloud-provider", "scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "73594": {
-    "commit": "af17925f68a1d43e91fc3d09a23b81b62c53409c",
-    "text": "Prevent AWS Network Load Balancer security groups ingress rules to be deleted by ensuring target groups are tagged.",
-    "markdown": "Prevent AWS Network Load Balancer security groups ingress rules to be deleted by ensuring target groups are tagged. ([#73594](https://github.com/kubernetes/kubernetes/pull/73594), [@masterzen](https://github.com/masterzen))",
-    "author": "masterzen",
-    "author_url": "https://github.com/masterzen",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73594",
-    "pr_number": 73594,
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.0"
-  },
-  "73595": {
-    "commit": "4992402fa5bee9f7ac6fe931d101d362a4295cea",
-    "text": "error messages returned in authentication webhook status responses are now correctly included in the apiserver log",
-    "markdown": "error messages returned in authentication webhook status responses are now correctly included in the apiserver log ([#73595](https://github.com/kubernetes/kubernetes/pull/73595), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73595",
-    "pr_number": 73595,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "73610": {
-    "commit": "b1eae78fc7b807391ecf017b8db1bbad68346af3",
-    "text": "CoreDNS is now version 1.3.1\n   - A new `k8s_external` plugin that allows external zones to point to Kubernetes in-cluster services.\n   - CoreDNS now checks if a zone transfer is allowed. Also allow a TTL of 0 to avoid caching in the cache plugin.\n   - TTL is also applied to negative responses (NXDOMAIN, etc).\n   - Also provides a deprecation notice for `upstream` where the upstream directive used in various plugin will start to default to the CoreDNS process itself.",
-    "markdown": "CoreDNS is now version 1.3.1\n     - A new `k8s_external` plugin that allows external zones to point to Kubernetes in-cluster services.\n     - CoreDNS now checks if a zone transfer is allowed. Also allow a TTL of 0 to avoid caching in the cache plugin.\n     - TTL is also applied to negative responses (NXDOMAIN, etc).\n     - Also provides a deprecation notice for `upstream` where the upstream directive used in various plugin will start to default to the CoreDNS process itself. ([#73610](https://github.com/kubernetes/kubernetes/pull/73610), [@rajansandeep](https://github.com/rajansandeep))",
-    "author": "rajansandeep",
-    "author_url": "https://github.com/rajansandeep",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73610",
-    "pr_number": 73610,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "73636": {
-    "commit": "b5b627d522ed74549e2583b258a7df0e3ffd33ea",
-    "text": "kubeadm reset: fixed crash caused by absence of a configuration file",
-    "markdown": "kubeadm reset: fixed crash caused by absence of a configuration file ([#73636](https://github.com/kubernetes/kubernetes/pull/73636), [@bart0sh](https://github.com/bart0sh))",
-    "author": "bart0sh",
-    "author_url": "https://github.com/bart0sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73636",
-    "pr_number": 73636,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "73651": {
-    "commit": "888ff4097a62d949f11b3c506b8768c6872ac4c2",
-    "text": "kubelet now accepts `pid=\u003cnumber\u003e` in the `--system-reserved` and `--kube-reserved` options to ensure that the specified number of process IDs will be reserved for the system as a whole and for Kubernetes system daemons respectively.  Please reference `Kube Reserved` and `System Reserved` in `Reserve Compute Resources for System Daemons` in the Kubernetes documentation for general discussion of resource reservation.  To utilize this functionality, you must set the feature gate `SupportNodePidsLimit=true`",
-    "markdown": "kubelet now accepts `pid=\u003cnumber\u003e` in the `--system-reserved` and `--kube-reserved` options to ensure that the specified number of process IDs will be reserved for the system as a whole and for Kubernetes system daemons respectively.  Please reference `Kube Reserved` and `System Reserved` in `Reserve Compute Resources for System Daemons` in the Kubernetes documentation for general discussion of resource reservation.  To utilize this functionality, you must set the feature gate `SupportNodePidsLimit=true` ([#73651](https://github.com/kubernetes/kubernetes/pull/73651), [@RobertKrawitz](https://github.com/RobertKrawitz))\n\n  Courtesy of SIG Architecture, SIG Node, and SIG Testing",
-    "author": "RobertKrawitz",
-    "author_url": "https://github.com/RobertKrawitz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73651",
-    "pr_number": 73651,
-    "areas": ["kubelet"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["architecture", "node", "testing"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "73653": {
-    "commit": "36787041cbb7a83531ffea765b045117f2fb301c",
-    "text": "Introduce dynamic volume provisioning shim for CSI migration",
-    "markdown": "Introduce dynamic volume provisioning shim for CSI migration ([#73653](https://github.com/kubernetes/kubernetes/pull/73653), [@ddebroy](https://github.com/ddebroy))\n\n  Courtesy of SIG Apps, and SIG Storage",
-    "author": "ddebroy",
-    "author_url": "https://github.com/ddebroy",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73653",
-    "pr_number": 73653,
-    "kinds": ["feature"],
-    "sigs": ["apps", "storage"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "73659": {
-    "commit": "ee44e24cd330ea07ca809fc600de2d10067155d0",
-    "text": "Kubelet: add usageNanoCores from CRI stats provider",
-    "markdown": "Kubelet: add usageNanoCores from CRI stats provider ([#73659](https://github.com/kubernetes/kubernetes/pull/73659), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73659",
-    "pr_number": 73659,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "73661": {
-    "commit": "9027738dd5b1a4a6d5a597d9487fdadad0e7ff71",
-    "text": "if smb mount failed:\n   if the original smb mount exists:\n      remove original smb mount\n      mount again",
-    "markdown": "if smb mount failed:\n     if the original smb mount exists:\n        remove original smb mount\n        mount again ([#73661](https://github.com/kubernetes/kubernetes/pull/73661), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73661",
-    "pr_number": 73661,
-    "kinds": ["bug"],
-    "sigs": ["windows"],
-    "release_version": "1.14.0"
-  },
-  "73663": {
-    "commit": "7e086471c40b3ea54e0c236f23693d35b02a080d",
-    "text": "kube-apiserver: the deprecated `repair-malformed-updates` has been removed",
-    "markdown": "kube-apiserver: the deprecated `repair-malformed-updates` has been removed ([#73663](https://github.com/kubernetes/kubernetes/pull/73663), [@danielqsj](https://github.com/danielqsj))",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73663",
-    "pr_number": 73663,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.0"
-  },
-  "73676": {
-    "commit": "38a325250fbefa8785740d00358978eefa160dde",
-    "text": "client-go: PortForwarder.GetPorts() now contain correct local port if no local port was initially specified when setting up the port forwarder",
-    "markdown": "client-go: PortForwarder.GetPorts() now contain correct local port if no local port was initially specified when setting up the port forwarder ([#73676](https://github.com/kubernetes/kubernetes/pull/73676), [@martin-helmich](https://github.com/martin-helmich))",
-    "author": "martin-helmich",
-    "author_url": "https://github.com/martin-helmich",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73676",
-    "pr_number": 73676,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.0"
-  },
-  "73700": {
-    "commit": "bf3b5e55634cac542e7cce16ba5844b067018bb4",
-    "text": "Don't update the Pod object after each scheduling attempt by adding a timestamp to the scheduling queue.",
-    "markdown": "Don't update the Pod object after each scheduling attempt by adding a timestamp to the scheduling queue. ([#73700](https://github.com/kubernetes/kubernetes/pull/73700), [@denkensk](https://github.com/denkensk))",
-    "author": "denkensk",
-    "author_url": "https://github.com/denkensk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73700",
-    "pr_number": 73700,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.0"
-  },
-  "73716": {
-    "commit": "fcb0d60d4c8d6018572b5e908ab933869e8b9067",
-    "text": "`kubectl delete --all-namespaces` is a recognized flag.",
-    "markdown": "`kubectl delete --all-namespaces` is a recognized flag. ([#73716](https://github.com/kubernetes/kubernetes/pull/73716), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73716",
-    "pr_number": 73716,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "73718": {
-    "commit": "1e78fec9b4db27396c0c4eefa15ea3a3f0b15306",
-    "text": "kubeadm: `kubeadm alpha preflight` and `kubeadm alpha preflight node` are removed; you can now use `kubeadm join phase preflight`",
-    "markdown": "kubeadm: `kubeadm alpha preflight` and `kubeadm alpha preflight node` are removed; you can now use `kubeadm join phase preflight` ([#73718](https://github.com/kubernetes/kubernetes/pull/73718), [@fabriziopandini](https://github.com/fabriziopandini))",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73718",
-    "pr_number": 73718,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "73721": {
-    "commit": "7788606ec65cedbf90a9749f02f941c5f6275125",
-    "text": "MAC Address filter has been fixed in vSphere Cloud Provider, it no longer ignores `00:1c:14` and `00:05:69` prefixes",
-    "markdown": "MAC Address filter has been fixed in vSphere Cloud Provider, it no longer ignores `00:1c:14` and `00:05:69` prefixes ([#73721](https://github.com/kubernetes/kubernetes/pull/73721), [@frapposelli](https://github.com/frapposelli))",
-    "author": "frapposelli",
-    "author_url": "https://github.com/frapposelli",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73721",
-    "pr_number": 73721,
-    "areas": ["provider/vmware"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.0"
-  },
-  "73726": {
-    "commit": "44d13d3b772db409e37711ca0c328bcb87f9f0c1",
-    "text": "Allow the kubelet to pass Windows GMSA credentials down to Docker",
-    "markdown": "Allow the kubelet to pass Windows GMSA credentials down to Docker ([#73726](https://github.com/kubernetes/kubernetes/pull/73726), [@wk8](https://github.com/wk8))\n\n  Courtesy of SIG Node, and SIG Windows",
-    "author": "wk8",
-    "author_url": "https://github.com/wk8",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73726",
-    "pr_number": 73726,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node", "windows"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "73758": {
-    "commit": "6796645672b4de8d2b6fae780c660926a23e326a",
-    "text": "Set a low oom_score_adj for containers in pods with system-critical priorities",
-    "markdown": "Set a low oom_score_adj for containers in pods with system-critical priorities ([#73758](https://github.com/kubernetes/kubernetes/pull/73758), [@sjenning](https://github.com/sjenning))",
-    "author": "sjenning",
-    "author_url": "https://github.com/sjenning",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73758",
-    "pr_number": 73758,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.0"
-  },
-  "73783": {
-    "commit": "3ecd17d2f928df180339be5237aa153563eb8227",
-    "text": "export query parameter is deprecated and will be removed in a future release",
-    "markdown": "export query parameter is deprecated and will be removed in a future release ([#73783](https://github.com/kubernetes/kubernetes/pull/73783), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73783",
-    "pr_number": 73783,
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.0"
-  },
-  "73787": {
-    "commit": "120bcd7ddd41e8145449a38c80b1f5557a4c7959",
-    "text": "Deprecate --export flag from kubectl get command.",
-    "markdown": "Deprecate --export flag from kubectl get command. ([#73787](https://github.com/kubernetes/kubernetes/pull/73787), [@soltysh](https://github.com/soltysh))",
-    "author": "soltysh",
-    "author_url": "https://github.com/soltysh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73787",
-    "pr_number": 73787,
-    "areas": ["kubectl"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "architecture", "cli"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "73802": {
-    "commit": "6a9902deee1af821ca6029ecca208d58c6532f6f",
-    "text": "Kubelet now tries to stop containers in unknown state once before restart or remove.",
-    "markdown": "Kubelet now tries to stop containers in unknown state once before restart or remove. ([#73802](https://github.com/kubernetes/kubernetes/pull/73802), [@Random-Liu](https://github.com/Random-Liu))",
-    "author": "Random-Liu",
-    "author_url": "https://github.com/Random-Liu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73802",
-    "pr_number": 73802,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.0"
-  },
-  "73807": {
-    "commit": "f16035600a4b9abf7e145f0ca2f1d1798c30c9a3",
-    "text": "Default RBAC policy no longer grants access to discovery and permission-checking APIs (used by `kubectl auth can-i`) to *unauthenticated- users. Upgraded clusters preserve prior behavior, but cluster administrators wishing to grant unauthenticated users access in new clusters will need to explicitly opt-in to expose the discovery and/or permission-checking APIs:\n- `kubectl create clusterrolebinding anonymous-discovery --clusterrole=system:discovery --group=system:unauthenticated`\n- `kubectl create clusterrolebinding anonymous-access-review --clusterrole=system:basic-user --group=system:unauthenticated`",
-    "markdown": "Default RBAC policy no longer grants access to discovery and permission-checking APIs (used by `kubectl auth can-i`) to *unauthenticated- users. Upgraded clusters preserve prior behavior, but cluster administrators wishing to grant unauthenticated users access in new clusters will need to explicitly opt-in to expose the discovery and/or permission-checking APIs:\n  - `kubectl create clusterrolebinding anonymous-discovery --clusterrole=system:discovery --group=system:unauthenticated`\n  - `kubectl create clusterrolebinding anonymous-access-review --clusterrole=system:basic-user --group=system:unauthenticated` ([#73807](https://github.com/kubernetes/kubernetes/pull/73807), [@dekkagaijin](https://github.com/dekkagaijin))\n\n  Courtesy of SIG Auth, and SIG Testing",
-    "author": "dekkagaijin",
-    "author_url": "https://github.com/dekkagaijin",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73807",
-    "pr_number": 73807,
-    "kinds": ["feature"],
-    "sigs": ["auth", "testing"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "73819": {
-    "commit": "743f8643103aff61d0e2db659e7c163eca8cc062",
-    "text": "Move fluentd-elasticsearch addon images to community controlled location",
-    "markdown": "Move fluentd-elasticsearch addon images to community controlled location ([#73819](https://github.com/kubernetes/kubernetes/pull/73819), [@coffeepac](https://github.com/coffeepac))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Instrumentation",
-    "author": "coffeepac",
-    "author_url": "https://github.com/coffeepac",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73819",
-    "pr_number": 73819,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "73824": {
-    "commit": "c88dcee3e94b6c8e3c30b67b0456d967a149e4dc",
-    "text": "Re-issue Allocate grpc calls before starting a container that requests device-plugin resources if the cached state is missing.",
-    "markdown": "Re-issue Allocate grpc calls before starting a container that requests device-plugin resources if the cached state is missing. ([#73824](https://github.com/kubernetes/kubernetes/pull/73824), [@jiayingz](https://github.com/jiayingz))",
-    "author": "jiayingz",
-    "author_url": "https://github.com/jiayingz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73824",
-    "pr_number": 73824,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.0"
-  },
-  "73833": {
-    "commit": "063e7ff358fdc8b0916e6f39beedc0d025734cb1",
-    "text": "[CRI] Add a new field called `runtime_handler` into PodSandbox and PodSandboxStatus to track the RuntimeClass information of a pod.",
-    "markdown": "[CRI] Add a new field called `runtime_handler` into PodSandbox and PodSandboxStatus to track the RuntimeClass information of a pod. ([#73833](https://github.com/kubernetes/kubernetes/pull/73833), [@haiyanmeng](https://github.com/haiyanmeng))",
-    "author": "haiyanmeng",
-    "author_url": "https://github.com/haiyanmeng",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73833",
-    "pr_number": 73833,
-    "areas": ["kubelet"],
-    "kinds": ["api-change"],
-    "sigs": ["node"],
-    "release_version": "1.14.0"
-  },
-  "73837": {
-    "commit": "a1cc48bf59dd456dcc60f67d005143fa9f33fbcc",
-    "text": "kubeadm: add a preflight check that throws a warning if the cgroup driver for Docker on Linux is not \"systemd\" as per the k8s.io CRI installation guide.",
-    "markdown": "kubeadm: add a preflight check that throws a warning if the cgroup driver for Docker on Linux is not \"systemd\" as per the k8s.io CRI installation guide. ([#73837](https://github.com/kubernetes/kubernetes/pull/73837), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73837",
-    "pr_number": 73837,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "73845": {
-    "commit": "fd633d192f6c09c46368f4300af152216b1089fc",
-    "text": "Fix watch to not send the same set of events multiple times causing watcher to go back in time",
-    "markdown": "Fix watch to not send the same set of events multiple times causing watcher to go back in time ([#73845](https://github.com/kubernetes/kubernetes/pull/73845), [@wojtek-t](https://github.com/wojtek-t))",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73845",
-    "pr_number": 73845,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "73857": {
-    "commit": "d81daec3c41973c4d299b97045bbe2a985b62779",
-    "text": "kubeadm: all master components are now exclusively relying on the `PriorityClassName` pod spec for annotating them as cluster critical components. Since `scheduler.alpha.kubernetes.io/critical-pod` annotation is no longer supported by Kubernetes 1.14 this annotation is no longer added to master components.",
-    "markdown": "kubeadm: all master components are now exclusively relying on the `PriorityClassName` pod spec for annotating them as cluster critical components. Since `scheduler.alpha.kubernetes.io/critical-pod` annotation is no longer supported by Kubernetes 1.14 this annotation is no longer added to master components. ([#73857](https://github.com/kubernetes/kubernetes/pull/73857), [@ereslibre](https://github.com/ereslibre))",
-    "author": "ereslibre",
-    "author_url": "https://github.com/ereslibre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73857",
-    "pr_number": 73857,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "73892": {
-    "commit": "19e57c616c9e3b12cf518c7246e6f12e1ce1daba",
-    "text": "Events reported for container creation, start, and stop now report the container name in the message and are more consistently formatted.",
-    "markdown": "Events reported for container creation, start, and stop now report the container name in the message and are more consistently formatted. ([#73892](https://github.com/kubernetes/kubernetes/pull/73892), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73892",
-    "pr_number": 73892,
-    "areas": ["kubelet"],
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["node", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "73898": {
-    "commit": "461dccf2461139ff135704ef861992de92e2f38a",
-    "text": "Is -\u003eIt in line 6",
-    "markdown": "Is -\u003eIt in line 6 ([#73898](https://github.com/kubernetes/kubernetes/pull/73898), [@xiezongzhe](https://github.com/xiezongzhe))",
-    "author": "xiezongzhe",
-    "author_url": "https://github.com/xiezongzhe",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73898",
-    "pr_number": 73898,
-    "kinds": ["cleanup"],
-    "sigs": ["testing"],
-    "release_version": "1.14.0"
-  },
-  "73907": {
-    "commit": "b4a2b63561903db491d0e07c8fba752c1709efce",
-    "text": "kubeadm: Allow to upload certificates required to join a new control-plane to kubeadm-certs secret using the flag `--experimental-upload-certs` on `init` or upload-certs phase.",
-    "markdown": "kubeadm: Allow to upload certificates required to join a new control-plane to kubeadm-certs secret using the flag `--experimental-upload-certs` on `init` or upload-certs phase. ([#73907](https://github.com/kubernetes/kubernetes/pull/73907), [@yagonobre](https://github.com/yagonobre))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "yagonobre",
-    "author_url": "https://github.com/yagonobre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73907",
-    "pr_number": 73907,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "73941": {
-    "commit": "6949a81ce678077fee7b44c1afb054f78e47217a",
-    "text": "kubeadm: fix a bug in the underlying library for diff related to characters like '%'",
-    "markdown": "kubeadm: fix a bug in the underlying library for diff related to characters like '%' ([#73941](https://github.com/kubernetes/kubernetes/pull/73941), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73941",
-    "pr_number": 73941,
-    "areas": ["apiserver", "kubeadm"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery", "architecture", "cli", "cluster-lifecycle"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "73946": {
-    "commit": "535064773ac79580fde260a41a979af5a292e7b2",
-    "text": "Add a new kubelet endpoint for serving first-class resource metrics",
-    "markdown": "Add a new kubelet endpoint for serving first-class resource metrics ([#73946](https://github.com/kubernetes/kubernetes/pull/73946), [@dashpole](https://github.com/dashpole))\n\n  Courtesy of SIG Node, and SIG Testing",
-    "author": "dashpole",
-    "author_url": "https://github.com/dashpole",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73946",
-    "pr_number": 73946,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node", "testing"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "73951": {
-    "commit": "dad8bc8a4ea044052bf9d0d24ceb679e826ffcf2",
-    "text": "kubeadm no longer dumps backtrace if it fails to remove the running containers on reset.",
-    "markdown": "kubeadm no longer dumps backtrace if it fails to remove the running containers on reset. ([#73951](https://github.com/kubernetes/kubernetes/pull/73951), [@rosti](https://github.com/rosti))",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73951",
-    "pr_number": 73951,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "73998": {
-    "commit": "2bfbbc31417b0d153cad01f3e5689b093546a37a",
-    "text": "kubeadm: allow the usage of --kubeconfig-dir and --config flags on kubeadm init",
-    "markdown": "kubeadm: allow the usage of --kubeconfig-dir and --config flags on kubeadm init ([#73998](https://github.com/kubernetes/kubernetes/pull/73998), [@yagonobre](https://github.com/yagonobre))",
-    "author": "yagonobre",
-    "author_url": "https://github.com/yagonobre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73998",
-    "pr_number": 73998,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "74000": {
-    "commit": "508a4f75444d3ee9cfde20da8fa6c8a578b8e084",
-    "text": "The apiserver, including both the kube-apiserver and apiservers built with the generic apiserver library, will now return 413 RequestEntityTooLarge error if a json patch contains more than 10,000 operations.",
-    "markdown": "The apiserver, including both the kube-apiserver and apiservers built with the generic apiserver library, will now return 413 RequestEntityTooLarge error if a json patch contains more than 10,000 operations. ([#74000](https://github.com/kubernetes/kubernetes/pull/74000), [@caesarxuchao](https://github.com/caesarxuchao))",
-    "author": "caesarxuchao",
-    "author_url": "https://github.com/caesarxuchao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74000",
-    "pr_number": 74000,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74025": {
-    "commit": "8d69dc630ba66d05ad95146583a7482ed1eb3f82",
-    "text": "kubeadm no longer allows using v1alpha3 configs for anything else than converting them to v1beta1.",
-    "markdown": "kubeadm no longer allows using v1alpha3 configs for anything else than converting them to v1beta1. ([#74025](https://github.com/kubernetes/kubernetes/pull/74025), [@rosti](https://github.com/rosti))",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74025",
-    "pr_number": 74025,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "74033": {
-    "commit": "43616fc920c195a6f892397d4b9b3f53c7aae1fa",
-    "text": "kubeadm: improved RequiredIPVSKernelModulesAvailable warning message",
-    "markdown": "kubeadm: improved RequiredIPVSKernelModulesAvailable warning message ([#74033](https://github.com/kubernetes/kubernetes/pull/74033), [@bart0sh](https://github.com/bart0sh))",
-    "author": "bart0sh",
-    "author_url": "https://github.com/bart0sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74033",
-    "pr_number": 74033,
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.14.0"
-  },
-  "74040": {
-    "commit": "cc8afb25fcfbb2a12ef43300db89d724b284e3e3",
-    "text": "Add ResourceVersion as a precondition for delete in order to ensure a delete fails if an unobserved change happens to an object.",
-    "markdown": "Add ResourceVersion as a precondition for delete in order to ensure a delete fails if an unobserved change happens to an object. ([#74040](https://github.com/kubernetes/kubernetes/pull/74040), [@ajatprabha](https://github.com/ajatprabha))\n\n  Courtesy of SIG API Machinery",
-    "author": "ajatprabha",
-    "author_url": "https://github.com/ajatprabha",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74040",
-    "pr_number": 74040,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74041": {
-    "commit": "50328e0ba70b98dcd30665a4f758504055d6bed6",
-    "text": "Optimize scheduler cache snapshot algorithm to improve scheduling throughput.",
-    "markdown": "Optimize scheduler cache snapshot algorithm to improve scheduling throughput. ([#74041](https://github.com/kubernetes/kubernetes/pull/74041), [@bsalamat](https://github.com/bsalamat))\n\n  Courtesy of SIG Scheduling",
-    "author": "bsalamat",
-    "author_url": "https://github.com/bsalamat",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74041",
-    "pr_number": 74041,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74057": {
-    "commit": "7d75b73e1d72cef7a0f0e2804f8e0a582ccb6b61",
-    "text": "The Ingress API is now available via `networking.k8s.io/v1beta1`. `extensions/v1beta1` Ingress objects are deprecated and will no longer be served in v1.18.",
-    "markdown": "The Ingress API is now available via `networking.k8s.io/v1beta1`. `extensions/v1beta1` Ingress objects are deprecated and will no longer be served in v1.18. ([#74057](https://github.com/kubernetes/kubernetes/pull/74057), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Architecture, SIG CLI, SIG Cloud Provider, SIG Network, and SIG Testing",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74057",
-    "pr_number": 74057,
-    "areas": ["apiserver", "code-generation", "kubectl"],
-    "kinds": ["api-change", "feature"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "architecture",
-      "cli",
-      "cloud-provider",
-      "network",
-      "testing"
-    ],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74065": {
-    "commit": "fd7acc3275f6ba4871d7e245e3bcf8a7ac563d1c",
-    "text": "Split up the mondo `kubernetes-test` tarball into `kubernetes-test-portable` and `kubernetes-test-{OS}-{ARCH}` tarballs.",
-    "markdown": "Split up the mondo `kubernetes-test` tarball into `kubernetes-test-portable` and `kubernetes-test-{OS}-{ARCH}` tarballs. ([#74065](https://github.com/kubernetes/kubernetes/pull/74065), [@ixdy](https://github.com/ixdy))",
-    "author": "ixdy",
-    "author_url": "https://github.com/ixdy",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74065",
-    "pr_number": 74065,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "release", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74086": {
-    "commit": "83ff0f6c64485ffe491d5116596072f466be1bd5",
-    "text": "Alpha support for ephemeral CSI inline volumes that are embedded in pod specs.",
-    "markdown": "Alpha support for ephemeral CSI inline volumes that are embedded in pod specs. ([#74086](https://github.com/kubernetes/kubernetes/pull/74086), [@vladimirvivien](https://github.com/vladimirvivien))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Architecture, SIG Auth, SIG CLI, and SIG Storage",
-    "author": "vladimirvivien",
-    "author_url": "https://github.com/vladimirvivien",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74086",
-    "pr_number": 74086,
-    "areas": ["apiserver", "kubectl"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "architecture", "auth", "cli", "storage"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74093": {
-    "commit": "042f9ed3af3929227666b56d997d11a149eeb0ee",
-    "text": "Reduces the cache TTL for negative responses to 5s minimum.",
-    "markdown": "Reduces the cache TTL for negative responses to 5s minimum. ([#74093](https://github.com/kubernetes/kubernetes/pull/74093), [@blakebarnett](https://github.com/blakebarnett))",
-    "author": "blakebarnett",
-    "author_url": "https://github.com/blakebarnett",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74093",
-    "pr_number": 74093,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74112": {
-    "commit": "9e53b85d28d33077e8bd377ba6ca5ced99391218",
-    "text": "kubeadm: remove local etcd members from the etcd cluster when kubeadm reset",
-    "markdown": "kubeadm: remove local etcd members from the etcd cluster when kubeadm reset ([#74112](https://github.com/kubernetes/kubernetes/pull/74112), [@pytimer](https://github.com/pytimer))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "pytimer",
-    "author_url": "https://github.com/pytimer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74112",
-    "pr_number": 74112,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74137": {
-    "commit": "7b203c6809a5af940dbd8962b2dc65f93b33b401",
-    "text": "CoreDNS adds readinessProbe which prevents loadbalancing to unready pods, and also allows rolling updates to work as expected.",
-    "markdown": "CoreDNS adds readinessProbe which prevents loadbalancing to unready pods, and also allows rolling updates to work as expected. ([#74137](https://github.com/kubernetes/kubernetes/pull/74137), [@rajansandeep](https://github.com/rajansandeep))",
-    "author": "rajansandeep",
-    "author_url": "https://github.com/rajansandeep",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74137",
-    "pr_number": 74137,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "74140": {
-    "commit": "81e6407393aa46f2695e71a015f93819f1df424c",
-    "text": "The examples in kubectl apply/get/delete are updated to support `-k` which uses a `kustomization.yaml` file",
-    "markdown": "The examples in kubectl apply/get/delete are updated to support `-k` which uses a `kustomization.yaml` file ([#74140](https://github.com/kubernetes/kubernetes/pull/74140), [@Liujingfang1](https://github.com/Liujingfang1))\n\n  Courtesy of SIG Auth, SIG CLI, SIG Release, and SIG Testing",
-    "author": "Liujingfang1",
-    "author_url": "https://github.com/Liujingfang1",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74140",
-    "pr_number": 74140,
-    "areas": ["kubectl", "release-eng"],
-    "kinds": ["feature"],
-    "sigs": ["auth", "cli", "release", "testing"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74154": {
-    "commit": "0ffd59e403ab1ce625d7a9b78ef6c6c295eb9e2b",
-    "text": "Fixes use of webhook admission plugins with multi-version custom resources",
-    "markdown": "Fixes use of webhook admission plugins with multi-version custom resources ([#74154](https://github.com/kubernetes/kubernetes/pull/74154), [@mbohlool](https://github.com/mbohlool))",
-    "author": "mbohlool",
-    "author_url": "https://github.com/mbohlool",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74154",
-    "pr_number": 74154,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74156": {
-    "commit": "f99728e6f35c3f2c52c338c123c9873e372ca945",
-    "text": "kubectl: fix a bug where \"describe\" cannot obtain the event messages for a static pod",
-    "markdown": "kubectl: fix a bug where \"describe\" cannot obtain the event messages for a static pod ([#74156](https://github.com/kubernetes/kubernetes/pull/74156), [@gaorong](https://github.com/gaorong))",
-    "author": "gaorong",
-    "author_url": "https://github.com/gaorong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74156",
-    "pr_number": 74156,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.0"
-  },
-  "74168": {
-    "commit": "c6b2a87cdb2f2e489cff2b2c0ba15e067fab9597",
-    "text": "kubeadm: Allow to download certificate secrets uploaded by `init` or `upload-certs` phase, allowing to transfer certificate secrets (certificates and keys) from the cluster to other master machines when creating HA deployments.",
-    "markdown": "kubeadm: Allow to download certificate secrets uploaded by `init` or `upload-certs` phase, allowing to transfer certificate secrets (certificates and keys) from the cluster to other master machines when creating HA deployments. ([#74168](https://github.com/kubernetes/kubernetes/pull/74168), [@ereslibre](https://github.com/ereslibre))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "ereslibre",
-    "author_url": "https://github.com/ereslibre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74168",
-    "pr_number": 74168,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74191": {
-    "commit": "8d6f20ea4342327bb778a80c70c47ec9ad47e182",
-    "text": "fix get azure accounts timeout issue when there is no out-bound IP",
-    "markdown": "fix get azure accounts timeout issue when there is no out-bound IP ([#74191](https://github.com/kubernetes/kubernetes/pull/74191), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74191",
-    "pr_number": 74191,
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74200": {
-    "commit": "eae5b730c859aaab738629dcc5567bb3d71726ba",
-    "text": "fix mixed protocol issue for azure load balancer",
-    "markdown": "fix mixed protocol issue for azure load balancer ([#74200](https://github.com/kubernetes/kubernetes/pull/74200), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74200",
-    "pr_number": 74200,
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.0"
-  },
-  "74222": {
-    "commit": "98fa2c7d32e59f798ff4201dfdcb439554392beb",
-    "text": "Kubelet won't evict a static pod with priority `system-node-critical` upon resource pressure.",
-    "markdown": "Kubelet won't evict a static pod with priority `system-node-critical` upon resource pressure. ([#74222](https://github.com/kubernetes/kubernetes/pull/74222), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74222",
-    "pr_number": 74222,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74267": {
-    "commit": "12647fa068e8c8aa8e3bbeac90fc1a64423bb54e",
-    "text": "kubelet's --containerized flag will no longer be supported and will be removed in a future release",
-    "markdown": "kubelet's --containerized flag will no longer be supported and will be removed in a future release ([#74267](https://github.com/kubernetes/kubernetes/pull/74267), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74267",
-    "pr_number": 74267,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.14.0"
-  },
-  "74283": {
-    "commit": "6c31101257bfcd47fa53702cea07fe2eedf2ad92",
-    "text": "Transition CSINodeInfo and CSIDriver alpha CRDs to in-tree CSINode and CSIDriver core storage v1beta1 APIs.\nthe alpha CRDs are no longer used and drivers will need to be updated to use the beta APIs.\nThe support for `_` in the CSI driver name will be dropped as the CSI Spec does not allow that.",
-    "markdown": "Transition CSINodeInfo and CSIDriver alpha CRDs to in-tree CSINode and CSIDriver core storage v1beta1 APIs.\n  the alpha CRDs are no longer used and drivers will need to be updated to use the beta APIs.\n  The support for `_` in the CSI driver name will be dropped as the CSI Spec does not allow that. ([#74283](https://github.com/kubernetes/kubernetes/pull/74283), [@xing-yang](https://github.com/xing-yang))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Architecture, SIG Auth, SIG Node, SIG Storage, and SIG Testing",
-    "author": "xing-yang",
-    "author_url": "https://github.com/xing-yang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74283",
-    "pr_number": 74283,
-    "areas": ["apiserver", "kubelet"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "architecture", "auth", "node", "storage", "testing"],
-    "feature": true,
-    "action_required": true,
-    "release_version": "1.14.0"
-  },
-  "74306": {
-    "commit": "ad1e021dcd7d1fd00e53422adee8c7a99ed6451d",
-    "text": "Fixed scanning of failed iSCSI targets.",
-    "markdown": "Fixed scanning of failed iSCSI targets. ([#74306](https://github.com/kubernetes/kubernetes/pull/74306), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74306",
-    "pr_number": 74306,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.0"
-  },
-  "74311": {
-    "commit": "d5a3db003916b1d33b503ccd2e4897e094d8af77",
-    "text": "Ensure ownership when deleting a load balancer security group",
-    "markdown": "Ensure ownership when deleting a load balancer security group ([#74311](https://github.com/kubernetes/kubernetes/pull/74311), [@hpedrorodrigues](https://github.com/hpedrorodrigues))",
-    "author": "hpedrorodrigues",
-    "author_url": "https://github.com/hpedrorodrigues",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74311",
-    "pr_number": 74311,
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.0"
-  },
-  "74348": {
-    "commit": "d9f3e96796e2d154b1d4caa156ba95ff9b01e5b2",
-    "text": "Fix keymutex issues which may crash in some platforms.",
-    "markdown": "Fix keymutex issues which may crash in some platforms. ([#74348](https://github.com/kubernetes/kubernetes/pull/74348), [@danielqsj](https://github.com/danielqsj))",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74348",
-    "pr_number": 74348,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "release", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74355": {
-    "commit": "9dbc8912d9ab607014ba8a5bd085b8c658106867",
-    "text": "kubeadm: Do not fail preflight checks when running on \u003e= 5.0 Linux kernel",
-    "markdown": "kubeadm: Do not fail preflight checks when running on \u003e= 5.0 Linux kernel ([#74355](https://github.com/kubernetes/kubernetes/pull/74355), [@brb](https://github.com/brb))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "brb",
-    "author_url": "https://github.com/brb",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74355",
-    "pr_number": 74355,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74376": {
-    "commit": "349ce4c3123de49debb73bb5eb29e625ee2711bb",
-    "text": "- Add duration metric for CRD webhook converters",
-    "markdown": "- Add duration metric for CRD webhook converters ([#74376](https://github.com/kubernetes/kubernetes/pull/74376), [@mbohlool](https://github.com/mbohlool))",
-    "author": "mbohlool",
-    "author_url": "https://github.com/mbohlool",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74376",
-    "pr_number": 74376,
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "instrumentation", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74389": {
-    "commit": "b77e4e24170104a2b9a32b46e9ad8a40b22ff676",
-    "text": "Fix the unexpected NotReady status when Node's iops is full if the runtime is dockershim.",
-    "markdown": "Fix the unexpected NotReady status when Node's iops is full if the runtime is dockershim. ([#74389](https://github.com/kubernetes/kubernetes/pull/74389), [@answer1991](https://github.com/answer1991))",
-    "author": "answer1991",
-    "author_url": "https://github.com/answer1991",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74389",
-    "pr_number": 74389,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["network", "node"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74394": {
-    "commit": "f33e5e8f7a3776cb7cedf44d12bc96ca6a8ebe4f",
-    "text": "Services of type=LoadBalancer which have no endpoints will now immediately ICMP reject connections, rather than time out.",
-    "markdown": "Services of type=LoadBalancer which have no endpoints will now immediately ICMP reject connections, rather than time out. ([#74394](https://github.com/kubernetes/kubernetes/pull/74394), [@thockin](https://github.com/thockin))",
-    "author": "thockin",
-    "author_url": "https://github.com/thockin",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74394",
-    "pr_number": 74394,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["network", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74398": {
-    "commit": "d5007409f7e4a8a4f322dfeb26b3eb61811c659d",
-    "text": "fix issue: fail to detach azure disk when there is server side error",
-    "markdown": "fix issue: fail to detach azure disk when there is server side error ([#74398](https://github.com/kubernetes/kubernetes/pull/74398), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74398",
-    "pr_number": 74398,
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.0"
-  },
-  "74418": {
-    "commit": "9b8c58644a51fb79386c11bf6b4a31cbbad8417e",
-    "text": "Convert `latency`/`latencies` in metrics name to `duration`.\nThe following metrics are changed and mark previous metrics as deprecated:\n- `rest_client_request_latency_seconds` -\u003e `rest_client_request_duration_seconds`\n- `apiserver_proxy_tunnel_sync_latency_secs` -\u003e `apiserver_proxy_tunnel_sync_duration_seconds`\n- `scheduler_scheduling_latency_seconds` -\u003e `scheduler_scheduling_duration_seconds `",
-    "markdown": "Convert `latency`/`latencies` in metrics name to `duration`.\n  The following metrics are changed and mark previous metrics as deprecated:\n  - `rest_client_request_latency_seconds` -\u003e `rest_client_request_duration_seconds`\n  - `apiserver_proxy_tunnel_sync_latency_secs` -\u003e `apiserver_proxy_tunnel_sync_duration_seconds`\n  - `scheduler_scheduling_latency_seconds` -\u003e `scheduler_scheduling_duration_seconds ` ([#74418](https://github.com/kubernetes/kubernetes/pull/74418), [@danielqsj](https://github.com/danielqsj))\n\n  Courtesy of SIG API Machinery, SIG Instrumentation, SIG Network, SIG Node, SIG Scheduling, and SIG Testing",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74418",
-    "pr_number": 74418,
-    "areas": ["apiserver", "kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "instrumentation", "network", "node", "scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74422": {
-    "commit": "be8a9b96377973f0a956a11d55be4929bbf9ffd0",
-    "text": "client-go: the deprecated versionless API group accessors (like `clientset.Apps()` have been removed). Use an explicit version instead (like `clientset.AppsV1()`)",
-    "markdown": "client-go: the deprecated versionless API group accessors (like `clientset.Apps()` have been removed). Use an explicit version instead (like `clientset.AppsV1()`) ([#74422](https://github.com/kubernetes/kubernetes/pull/74422), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74422",
-    "pr_number": 74422,
-    "areas": ["code-generation", "kubeadm", "kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "auth",
-      "autoscaling",
-      "cli",
-      "cluster-lifecycle",
-      "storage",
-      "testing"
-    ],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74424": {
-    "commit": "45e5f6053b0670fc776b8a2e7ccb4b1d49445ba5",
-    "text": "- the fluentd addon daemonset will now target all nodes.\n- setting `ENABLE_METADATA_CONCEALMENT=true` in kube-up will now set a `cloud.google.com/metadata-proxy-ready=true` label on new nodes. In v1.16, the metadata proxy add-on will switch to using that label as a node selector.\n- setting `KUBE_PROXY_DAEMONSET=true` in kube-up will now set a `node.kubernetes.io/kube-proxy-ds-ready=true` label on new nodes. In v1.16, the kube-proxy daemonset add-on will switch to using that label as a node selector.\n- In 1.16, the masq-agent daemonset add-on will switch to using `node.kubernetes.io/masq-agent-ds-ready` as a node selector.",
-    "markdown": "- the fluentd addon daemonset will now target all nodes.\n  - setting `ENABLE_METADATA_CONCEALMENT=true` in kube-up will now set a `cloud.google.com/metadata-proxy-ready=true` label on new nodes. In v1.16, the metadata proxy add-on will switch to using that label as a node selector.\n  - setting `KUBE_PROXY_DAEMONSET=true` in kube-up will now set a `node.kubernetes.io/kube-proxy-ds-ready=true` label on new nodes. In v1.16, the kube-proxy daemonset add-on will switch to using that label as a node selector.\n  - In 1.16, the masq-agent daemonset add-on will switch to using `node.kubernetes.io/masq-agent-ds-ready` as a node selector. ([#74424](https://github.com/kubernetes/kubernetes/pull/74424), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74424",
-    "pr_number": 74424,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "instrumentation", "node"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74433": {
-    "commit": "183e6cc1902c2088a40e60cf4138bd829d4ad80f",
-    "text": "The node.k8s.io API group and runtimeclasses.node.k8s.io resource have been migrated to a built-in API. If you were using RuntimeClasses (an default-disabled alpha feature, as of Kubernetes v1.12), then you must recreate all RuntimeClasses after upgrading, and the runtimeclasses.node.k8s.io CRD should be manually deleted. RuntimeClasses can no longer be created without a defined handler.\n\nIntroduce a RuntimeClass v1beta1 API. This new beta API renames `runtimeHandler` to `handler`, makes it a required field, and cuts out the spec (handler is a top-level field).",
-    "markdown": "The node.k8s.io API group and runtimeclasses.node.k8s.io resource have been migrated to a built-in API. If you were using RuntimeClasses (an default-disabled alpha feature, as of Kubernetes v1.12), then you must recreate all RuntimeClasses after upgrading, and the runtimeclasses.node.k8s.io CRD should be manually deleted. RuntimeClasses can no longer be created without a defined handler.\n  \n  Introduce a RuntimeClass v1beta1 API. This new beta API renames `runtimeHandler` to `handler`, makes it a required field, and cuts out the spec (handler is a top-level field). ([#74433](https://github.com/kubernetes/kubernetes/pull/74433), [@tallclair](https://github.com/tallclair))\n\n  Courtesy of SIG API Machinery, SIG Architecture, SIG CLI, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Node, and SIG Testing",
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74433",
-    "pr_number": 74433,
-    "areas": ["apiserver", "kubelet"],
-    "kinds": ["api-change", "feature"],
-    "sigs": [
-      "api-machinery",
-      "architecture",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "node",
-      "testing"
-    ],
-    "feature": true,
-    "action_required": true,
-    "release_version": "1.14.0"
-  },
-  "74434": {
-    "commit": "1fc2396e2e96360a33341739ef82ae956d0c9862",
-    "text": "PodReadinessGate feature is now GA. The feature gate will not allow disabling it.",
-    "markdown": "PodReadinessGate feature is now GA. The feature gate will not allow disabling it. ([#74434](https://github.com/kubernetes/kubernetes/pull/74434), [@freehan](https://github.com/freehan))\n\n  Courtesy of SIG Network",
-    "author": "freehan",
-    "author_url": "https://github.com/freehan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74434",
-    "pr_number": 74434,
-    "kinds": ["feature"],
-    "sigs": ["network"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74441": {
-    "commit": "b15056010789632ae0f905f0919e514c8ea23c02",
-    "text": "Change CRI pod log directory from `/var/log/pods/UID` to `/var/log/pods/NAMESPACE_NAME_UID`.\n\nIt is recommended to drain the node before upgrade, or reboot the node after upgrade.",
-    "markdown": "Change CRI pod log directory from `/var/log/pods/UID` to `/var/log/pods/NAMESPACE_NAME_UID`.\n  \n  It is recommended to drain the node before upgrade, or reboot the node after upgrade. ([#74441](https://github.com/kubernetes/kubernetes/pull/74441), [@Random-Liu](https://github.com/Random-Liu))\n\n  Courtesy of SIG Cluster Lifecycle, SIG Node, and SIG Testing",
-    "author": "Random-Liu",
-    "author_url": "https://github.com/Random-Liu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74441",
-    "pr_number": 74441,
-    "areas": ["kubelet", "test"],
-    "kinds": ["bug", "cleanup", "feature"],
-    "sigs": ["cluster-lifecycle", "node", "testing"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74448": {
-    "commit": "7d67315d28f51323b634cffcc32833ae0a1bf14b",
-    "text": "Fixes an issue with missing apiVersion/kind in object data sent to admission webhooks",
-    "markdown": "Fixes an issue with missing apiVersion/kind in object data sent to admission webhooks ([#74448](https://github.com/kubernetes/kubernetes/pull/74448), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74448",
-    "pr_number": 74448,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.0"
-  },
-  "74454": {
-    "commit": "1cfaf2bdc048d96b1125a5f733f0de37d329f27c",
-    "text": "kubeadm: fixed nil pointer dereference caused by a bug in url parsing",
-    "markdown": "kubeadm: fixed nil pointer dereference caused by a bug in url parsing ([#74454](https://github.com/kubernetes/kubernetes/pull/74454), [@bart0sh](https://github.com/bart0sh))",
-    "author": "bart0sh",
-    "author_url": "https://github.com/bart0sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74454",
-    "pr_number": 74454,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "74465": {
-    "commit": "c9e14c85f5b826493a2376ab4743499ac834b8ee",
-    "text": "PriorityClass in scheduling.k8s.io/v1beta1 and scheduling.k8s.io/v1alpha1 are deprecated by PriorityClass scheduling.k8s.io/v1 and will not be served starting in v1.17.",
-    "markdown": "PriorityClass in scheduling.k8s.io/v1beta1 and scheduling.k8s.io/v1alpha1 are deprecated by PriorityClass scheduling.k8s.io/v1 and will not be served starting in v1.17. ([#74465](https://github.com/kubernetes/kubernetes/pull/74465), [@bsalamat](https://github.com/bsalamat))",
-    "author": "bsalamat",
-    "author_url": "https://github.com/bsalamat",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74465",
-    "pr_number": 74465,
-    "kinds": ["api-change"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.0"
-  },
-  "74477": {
-    "commit": "8b052158c7381807e7dde41f7ff989e71093676e",
-    "text": "Admission webhooks rules can now limit scope to only match namespaced, or only cluster-scoped resources with a `scope: \"Cluster\" | \"Namespaced\" | \"*\"` field.",
-    "markdown": "Admission webhooks rules can now limit scope to only match namespaced, or only cluster-scoped resources with a `scope: \"Cluster\" | \"Namespaced\" | \"*\"` field. ([#74477](https://github.com/kubernetes/kubernetes/pull/74477), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74477",
-    "pr_number": 74477,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74498": {
-    "commit": "3c92a6d34410f65db7b80a8b195986fda240238a",
-    "text": "Fix subnet annotation checking for Azure internal loadbalancer",
-    "markdown": "Fix subnet annotation checking for Azure internal loadbalancer ([#74498](https://github.com/kubernetes/kubernetes/pull/74498), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74498",
-    "pr_number": 74498,
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.0"
-  },
-  "74499": {
-    "commit": "b3ca8262bebefcf6753a456fb2c0bb8548c2ad26",
-    "text": "fix parse devicePath issue on Azure Disk",
-    "markdown": "fix parse devicePath issue on Azure Disk ([#74499](https://github.com/kubernetes/kubernetes/pull/74499), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74499",
-    "pr_number": 74499,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.0"
-  },
-  "74502": {
-    "commit": "a213886f9cd37ea9bc4c7b2fdbdb872f86880570",
-    "text": "Support collecting pod logs under /var/log/pods/NAMESPACE_NAME_UID to stackdriver with `k8s_pod` resource type.",
-    "markdown": "Support collecting pod logs under /var/log/pods/NAMESPACE_NAME_UID to stackdriver with `k8s_pod` resource type. ([#74502](https://github.com/kubernetes/kubernetes/pull/74502), [@Random-Liu](https://github.com/Random-Liu))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Instrumentation",
-    "author": "Random-Liu",
-    "author_url": "https://github.com/Random-Liu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74502",
-    "pr_number": 74502,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74520": {
-    "commit": "cd9e590178c9f3812e296484b587de1c79461033",
-    "text": "- Fixes a bug concerning Quobyte volumes where user mappings only worked if the hosts Kubernetes plugin mount was provided via an external configuration using the _allow-usermapping-in-volumename_ option.",
-    "markdown": "- Fixes a bug concerning Quobyte volumes where user mappings only worked if the hosts Kubernetes plugin mount was provided via an external configuration using the _allow-usermapping-in-volumename_ option. ([#74520](https://github.com/kubernetes/kubernetes/pull/74520), [@casusbelli](https://github.com/casusbelli))",
-    "author": "casusbelli",
-    "author_url": "https://github.com/casusbelli",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74520",
-    "pr_number": 74520,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.0"
-  },
-  "74522": {
-    "commit": "b8ddc7945b6c97e0f4b564819357784f57114269",
-    "text": "- Fix liveness probe in fluentd-gcp cluster addon",
-    "markdown": "- Fix liveness probe in fluentd-gcp cluster addon ([#74522](https://github.com/kubernetes/kubernetes/pull/74522), [@Pluies](https://github.com/Pluies))",
-    "author": "Pluies",
-    "author_url": "https://github.com/Pluies",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74522",
-    "pr_number": 74522,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "74529": {
-    "commit": "0a4308f641cb13e5eda47909736858596c063eca",
-    "text": "Fixes panic if a kubelet is run against an older kube-apiserver",
-    "markdown": "Fixes panic if a kubelet is run against an older kube-apiserver ([#74529](https://github.com/kubernetes/kubernetes/pull/74529), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74529",
-    "pr_number": 74529,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.0"
-  },
-  "74542": {
-    "commit": "6c5810c4957b9f47fd50240070c8e023a1774148",
-    "text": "Allow Cinder volume limit to be configured from node too",
-    "markdown": "Allow Cinder volume limit to be configured from node too ([#74542](https://github.com/kubernetes/kubernetes/pull/74542), [@gnufied](https://github.com/gnufied))\n\n  Courtesy of SIG Cloud Provider, and SIG Storage",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74542",
-    "pr_number": 74542,
-    "areas": ["provider/openstack"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider", "storage"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74544": {
-    "commit": "d5e4973a32fd52ac297700f71e31ad2e1aace0ef",
-    "text": "Deprecate AWS, Azure, GCE and Cinder specific volume limit predicates.",
-    "markdown": "Deprecate AWS, Azure, GCE and Cinder specific volume limit predicates. ([#74544](https://github.com/kubernetes/kubernetes/pull/74544), [@gnufied](https://github.com/gnufied))\n\n  Courtesy of SIG Scheduling, and SIG Storage",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74544",
-    "pr_number": 74544,
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "storage"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74562": {
-    "commit": "218642e84ed0e3fcfda1447c8564ba24b90c1f5a",
-    "text": "Add a configuration field to shorten the timeout of validating/mutating admission webhook call. The timeout value must be between 1 and 30 seconds. Default to 30 seconds when unspecified.",
-    "markdown": "Add a configuration field to shorten the timeout of validating/mutating admission webhook call. The timeout value must be between 1 and 30 seconds. Default to 30 seconds when unspecified. ([#74562](https://github.com/kubernetes/kubernetes/pull/74562), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74562",
-    "pr_number": 74562,
-    "areas": ["admission-control", "apiserver"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74596": {
-    "commit": "e476ab63cc3188703e3892cb21912f606be1d6eb",
-    "text": "The kube-apiserver OpenAPI definitions with the prefix \"io.k8s.kubernetes.pkg\" (deprecated since 1.9) have been removed.",
-    "markdown": "The kube-apiserver OpenAPI definitions with the prefix \"io.k8s.kubernetes.pkg\" (deprecated since 1.9) have been removed. ([#74596](https://github.com/kubernetes/kubernetes/pull/74596), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74596",
-    "pr_number": 74596,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.0"
-  },
-  "74611": {
-    "commit": "f6d05d6315990c2aae7e6dd77f4772707b0452b2",
-    "text": "fix the flake in scheduling_queue_test.go",
-    "markdown": "fix the flake in scheduling_queue_test.go ([#74611](https://github.com/kubernetes/kubernetes/pull/74611), [@denkensk](https://github.com/denkensk))",
-    "author": "denkensk",
-    "author_url": "https://github.com/denkensk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74611",
-    "pr_number": 74611,
-    "kinds": ["failing-test", "flake"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.0"
-  },
-  "74615": {
-    "commit": "dccb8ab5b02e44cf80bc3f7927fdd0254370d7f6",
-    "text": "Remove the out-of-tree PersistentVolumeLabel controller because it cannot run without Initializers (removed in v1.14). If you are using AWS EBS, GCE PD, Azure Disk, Cinder Disk or vSphere volumes and rely on zone labels, then enable the `PersistentVolumeLabel` admission controller in the `kube-apiserver` in the `--enable-admission-plugins` flag.",
-    "markdown": "Remove the out-of-tree PersistentVolumeLabel controller because it cannot run without Initializers (removed in v1.14). If you are using AWS EBS, GCE PD, Azure Disk, Cinder Disk or vSphere volumes and rely on zone labels, then enable the `PersistentVolumeLabel` admission controller in the `kube-apiserver` in the `--enable-admission-plugins` flag. ([#74615](https://github.com/kubernetes/kubernetes/pull/74615), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74615",
-    "pr_number": 74615,
-    "areas": ["apiserver", "cloudprovider"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "apps", "cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74632": {
-    "commit": "36e934d0a9a004a5f6c0a06e22e89ec4fe583b6a",
-    "text": "Update to use golang 1.12",
-    "markdown": "Update to use golang 1.12 ([#74632](https://github.com/kubernetes/kubernetes/pull/74632), [@cblecker](https://github.com/cblecker))\n\n  Courtesy of SIG API Machinery, SIG Release, SIG Scalability, and SIG Testing",
-    "author": "cblecker",
-    "author_url": "https://github.com/cblecker",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74632",
-    "pr_number": 74632,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "release", "scalability", "testing"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74636": {
-    "commit": "a514fa042e49c5b95d6c03351563e54e14bc5e97",
-    "text": "This PR removes the following metrics:\n\n  reflector_items_per_list\n  reflector_items_per_watch\n  reflector_last_resource_version\n  reflector_list_duration_seconds\n  reflector_lists_total\n  reflector_short_watches_total\n  reflector_watch_duration_seconds\n  reflector_watches_total\n\nWhile this is a backwards-incompatible change, it would have been impossible to setup reliable monitoring around these metrics since the labels were not stable.",
-    "markdown": "This PR removes the following metrics:\n  \n    reflector_items_per_list\n    reflector_items_per_watch\n    reflector_last_resource_version\n    reflector_list_duration_seconds\n    reflector_lists_total\n    reflector_short_watches_total\n    reflector_watch_duration_seconds\n    reflector_watches_total\n  \n  While this is a backwards-incompatible change, it would have been impossible to setup reliable monitoring around these metrics since the labels were not stable. ([#74636](https://github.com/kubernetes/kubernetes/pull/74636), [@logicalhan](https://github.com/logicalhan))",
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74636",
-    "pr_number": 74636,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74652": {
-    "commit": "8477c486a81c1094440cf7c78d6be5c52b0828c4",
-    "text": "Delay CSI client initialization to make reconstruction of CSI volume possible because clients may not be available on kubelet restart.",
-    "markdown": "Delay CSI client initialization to make reconstruction of CSI volume possible because clients may not be available on kubelet restart. ([#74652](https://github.com/kubernetes/kubernetes/pull/74652), [@cofyc](https://github.com/cofyc))",
-    "author": "cofyc",
-    "author_url": "https://github.com/cofyc",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74652",
-    "pr_number": 74652,
-    "areas": ["kubelet", "test"],
-    "kinds": ["failing-test", "flake"],
-    "sigs": ["node", "storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74653": {
-    "commit": "8dd60281ffd66608681c594f6a317444ee153532",
-    "text": "Because some plugins mount volume on sub-directory of volume path, we need to distinguish between volume path and mount path to fix issue in reconstruction.",
-    "markdown": "Because some plugins mount volume on sub-directory of volume path, we need to distinguish between volume path and mount path to fix issue in reconstruction. ([#74653](https://github.com/kubernetes/kubernetes/pull/74653), [@cofyc](https://github.com/cofyc))",
-    "author": "cofyc",
-    "author_url": "https://github.com/cofyc",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74653",
-    "pr_number": 74653,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74668": {
-    "commit": "b1d4d40679883d8605f959c7b09e9b0ba6dddcc8",
-    "text": "Fix kube-apiserver not to create default/kubernetes service endpoints before it reports readiness via the /healthz and therefore is ready to serve requests. Also early during startup old endpoints are remove which might be left over from a previously crashed kube-apiserver.",
-    "markdown": "Fix kube-apiserver not to create default/kubernetes service endpoints before it reports readiness via the /healthz and therefore is ready to serve requests. Also early during startup old endpoints are remove which might be left over from a previously crashed kube-apiserver. ([#74668](https://github.com/kubernetes/kubernetes/pull/74668), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74668",
-    "pr_number": 74668,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74671": {
-    "commit": "2b63efcd3e4b6db327e0bb6615819c837dede8c4",
-    "text": "kubeadm: add certificate-key and skip-certificate-key-print flags to kubeadm init",
-    "markdown": "kubeadm: add certificate-key and skip-certificate-key-print flags to kubeadm init ([#74671](https://github.com/kubernetes/kubernetes/pull/74671), [@yagonobre](https://github.com/yagonobre))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "yagonobre",
-    "author_url": "https://github.com/yagonobre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74671",
-    "pr_number": 74671,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74675": {
-    "commit": "f330633a275eab44963af178ce790f1583a93020",
-    "text": "- Kubelet: replace `du` and `find` with a golang implementation\n- Kubelet: periodically update machine info to support hot-add/remove",
-    "markdown": "- Kubelet: replace `du` and `find` with a golang implementation\n  - Kubelet: periodically update machine info to support hot-add/remove ([#74675](https://github.com/kubernetes/kubernetes/pull/74675), [@dashpole](https://github.com/dashpole))",
-    "author": "dashpole",
-    "author_url": "https://github.com/dashpole",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74675",
-    "pr_number": 74675,
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.0"
-  },
-  "74682": {
-    "commit": "ce4229f2cc4020d2bb4bfbd540008a064dc16299",
-    "text": "Fix --help flag parsing",
-    "markdown": "Fix --help flag parsing ([#74682](https://github.com/kubernetes/kubernetes/pull/74682), [@soltysh](https://github.com/soltysh))",
-    "author": "soltysh",
-    "author_url": "https://github.com/soltysh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74682",
-    "pr_number": 74682,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.0"
-  },
-  "74712": {
-    "commit": "7514c49ec12e7a8b6840142d723ec25b08a0917a",
-    "text": "Fix help message for --container-runtime-endpoint: only unix socket is support on Linux.",
-    "markdown": "Fix help message for --container-runtime-endpoint: only unix socket is support on Linux. ([#74712](https://github.com/kubernetes/kubernetes/pull/74712), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74712",
-    "pr_number": 74712,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.0"
-  },
-  "74715": {
-    "commit": "a5ba825b1b09afc172468a1c2756e3617d947c9e",
-    "text": "fix Azure Container Registry anonymous repo image pull error",
-    "markdown": "fix Azure Container Registry anonymous repo image pull error ([#74715](https://github.com/kubernetes/kubernetes/pull/74715), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74715",
-    "pr_number": 74715,
-    "kinds": ["bug"],
-    "release_version": "1.14.0"
-  },
-  "74720": {
-    "commit": "23a41c9d7eefbe2209446a640c7767d8f7d926ee",
-    "text": "The deprecated `MountPropagation` feature gate has been removed, and the feature is now unconditionally enabled.",
-    "markdown": "The deprecated `MountPropagation` feature gate has been removed, and the feature is now unconditionally enabled. ([#74720](https://github.com/kubernetes/kubernetes/pull/74720), [@bertinatto](https://github.com/bertinatto))",
-    "author": "bertinatto",
-    "author_url": "https://github.com/bertinatto",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74720",
-    "pr_number": 74720,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74731": {
-    "commit": "da5fd93e93f463e69c598e841915ff15396d5456",
-    "text": "kube-conformance image will now run ginkgo with the --dryRun flag if the container is run with the environment variable E2E_DRYRUN set.",
-    "markdown": "kube-conformance image will now run ginkgo with the --dryRun flag if the container is run with the environment variable E2E_DRYRUN set. ([#74731](https://github.com/kubernetes/kubernetes/pull/74731), [@johnSchnake](https://github.com/johnSchnake))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Release",
-    "author": "johnSchnake",
-    "author_url": "https://github.com/johnSchnake",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74731",
-    "pr_number": 74731,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "release"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74755": {
-    "commit": "d8d260a09d87efda1d5d492db8d9e19594245258",
-    "text": "kubelet: resolved hang/timeout issues when running large numbers of pods with unique configmap/secret references",
-    "markdown": "kubelet: resolved hang/timeout issues when running large numbers of pods with unique configmap/secret references ([#74755](https://github.com/kubernetes/kubernetes/pull/74755), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74755",
-    "pr_number": 74755,
-    "areas": ["kubeadm", "kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "node", "scalability"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74769": {
-    "commit": "cb89d8028ae0a8cad16f6628bfda952c4c3747a4",
-    "text": "The PersistentLocalVolumes feature is GA. The feature gate cannot be disabled and will be removed in Kubernetes 1.17",
-    "markdown": "The PersistentLocalVolumes feature is GA. The feature gate cannot be disabled and will be removed in Kubernetes 1.17 ([#74769](https://github.com/kubernetes/kubernetes/pull/74769), [@msau42](https://github.com/msau42))\n\n  Courtesy of SIG Storage",
-    "author": "msau42",
-    "author_url": "https://github.com/msau42",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74769",
-    "pr_number": 74769,
-    "kinds": ["feature"],
-    "sigs": ["storage"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74787": {
-    "commit": "b82868bb7eaf3643612c718bddca1d21abc1849a",
-    "text": "iscsi modules haven't even been loaded /sys/class/iscsi_host directory won't exist",
-    "markdown": "iscsi modules haven't even been loaded /sys/class/iscsi_host directory won't exist ([#74787](https://github.com/kubernetes/kubernetes/pull/74787), [@jianglingxia](https://github.com/jianglingxia))",
-    "author": "jianglingxia",
-    "author_url": "https://github.com/jianglingxia",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74787",
-    "pr_number": 74787,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.0"
-  },
-  "74788": {
-    "commit": "fbc5150c7d2eb6a71dcc77cd9d8e4ba6c900c44c",
-    "text": "Add network stats for Windows nodes and containers",
-    "markdown": "Add network stats for Windows nodes and containers ([#74788](https://github.com/kubernetes/kubernetes/pull/74788), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74788",
-    "pr_number": 74788,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74804": {
-    "commit": "d0c3b708026f681f7467d36dcf9696f89ccfcfab",
-    "text": "Add `nullable` support to CustomResourceDefinition OpenAPI validation schemata.",
-    "markdown": "Add `nullable` support to CustomResourceDefinition OpenAPI validation schemata. ([#74804](https://github.com/kubernetes/kubernetes/pull/74804), [@sttts](https://github.com/sttts))\n\n  Courtesy of SIG API Machinery",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74804",
-    "pr_number": 74804,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74806": {
-    "commit": "c54978a6cf9e70a878fc4f67e42f36f149f30330",
-    "text": "Increase api server client certificate expiration histogram resolution to accommodate short-lived (\u003c 6h) client certificates.",
-    "markdown": "Increase api server client certificate expiration histogram resolution to accommodate short-lived (\u003c 6h) client certificates. ([#74806](https://github.com/kubernetes/kubernetes/pull/74806), [@mxinden](https://github.com/mxinden))\n\n  Courtesy of SIG API Machinery, and SIG Auth",
-    "author": "mxinden",
-    "author_url": "https://github.com/mxinden",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74806",
-    "pr_number": 74806,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "auth"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74809": {
-    "commit": "243e7408853156057d0d913c1f13c42dc1c3b3e6",
-    "text": "Kubelet no longer watches configmaps and secrets for terminated pods, in worst scenario causing it to not be able to send other requests to kube-apiserver",
-    "markdown": "Kubelet no longer watches configmaps and secrets for terminated pods, in worst scenario causing it to not be able to send other requests to kube-apiserver ([#74809](https://github.com/kubernetes/kubernetes/pull/74809), [@oxddr](https://github.com/oxddr))",
-    "author": "oxddr",
-    "author_url": "https://github.com/oxddr",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74809",
-    "pr_number": 74809,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.0"
-  },
-  "74830": {
-    "commit": "5bc1e9b8ea2f93503d898d9e4e77357709f0aefe",
-    "text": "The CSIPersistentVolume and KubeletPluginWatcher feature gates cannot be disabled, and will be removed in Kubernetes v1.16",
-    "markdown": "The CSIPersistentVolume and KubeletPluginWatcher feature gates cannot be disabled, and will be removed in Kubernetes v1.16 ([#74830](https://github.com/kubernetes/kubernetes/pull/74830), [@msau42](https://github.com/msau42))",
-    "author": "msau42",
-    "author_url": "https://github.com/msau42",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74830",
-    "pr_number": 74830,
-    "kinds": ["cleanup"],
-    "sigs": ["storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74835": {
-    "commit": "ad27abde622f9d9dec9977466bb2c0dace4e30a3",
-    "text": "If CSINodeInfo and CSIMigration feature flags are active in the cluster, Kubelet will post NotReady until CSINode is initialized with basic volume plugin mechanism information for well-known drivers",
-    "markdown": "If CSINodeInfo and CSIMigration feature flags are active in the cluster, Kubelet will post NotReady until CSINode is initialized with basic volume plugin mechanism information for well-known drivers ([#74835](https://github.com/kubernetes/kubernetes/pull/74835), [@davidz627](https://github.com/davidz627))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG Node, SIG Storage, and SIG Testing",
-    "author": "davidz627",
-    "author_url": "https://github.com/davidz627",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74835",
-    "pr_number": 74835,
-    "areas": ["kubelet"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "auth", "node", "storage", "testing"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74837": {
-    "commit": "eb95c27a5574e6ac2b166f9a6942226087c351e0",
-    "text": "Pre-existing log files are now opened with O_APPEND, instead of O_TRUNC. This helps prevent losing logs when components crash-loop, and also enables external log rotation utilities to truncate log files in-place without components extending log files to their pre-truncation sizes on subsequent writes.",
-    "markdown": "Pre-existing log files are now opened with O_APPEND, instead of O_TRUNC. This helps prevent losing logs when components crash-loop, and also enables external log rotation utilities to truncate log files in-place without components extending log files to their pre-truncation sizes on subsequent writes. ([#74837](https://github.com/kubernetes/kubernetes/pull/74837), [@mtaufen](https://github.com/mtaufen))\n\n  Courtesy of SIG API Machinery, SIG Architecture, SIG CLI, SIG Cloud Provider, and SIG Cluster Lifecycle",
-    "author": "mtaufen",
-    "author_url": "https://github.com/mtaufen",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74837",
-    "pr_number": 74837,
-    "areas": ["apiserver", "code-generation"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "architecture", "cli", "cloud-provider", "cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74863": {
-    "commit": "3624c74ce880168eb456f15ef50a92d1cbff8da1",
-    "text": "Add support for node side CSI volume expansion",
-    "markdown": "Add support for node side CSI volume expansion ([#74863](https://github.com/kubernetes/kubernetes/pull/74863), [@gnufied](https://github.com/gnufied))\n\n  Courtesy of SIG Node, SIG Storage, and SIG Testing",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74863",
-    "pr_number": 74863,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node", "storage", "testing"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74882": {
-    "commit": "e330c0120eed4871d518522b3f5c049c36a92464",
-    "text": "The resource group name in Azure providerID is not converted to lower cases.",
-    "markdown": "The resource group name in Azure providerID is not converted to lower cases. ([#74882](https://github.com/kubernetes/kubernetes/pull/74882), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74882",
-    "pr_number": 74882,
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.0"
-  },
-  "74909": {
-    "commit": "6dbe98980f97d966cb940cffb3e371b9e43da400",
-    "text": "The CSIBlockVolume feature gate is now beta, and defaults to enabled.",
-    "markdown": "The CSIBlockVolume feature gate is now beta, and defaults to enabled. ([#74909](https://github.com/kubernetes/kubernetes/pull/74909), [@bswartz](https://github.com/bswartz))\n\n  Courtesy of SIG Testing",
-    "author": "bswartz",
-    "author_url": "https://github.com/bswartz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74909",
-    "pr_number": 74909,
-    "kinds": ["feature"],
-    "sigs": ["testing"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74936": {
-    "commit": "2f930306a435d5c07fdb870c6c9b43cae4350b75",
-    "text": "Fix kubelet start failure issue on Azure Stack due to InstanceMetadata setting",
-    "markdown": "Fix kubelet start failure issue on Azure Stack due to InstanceMetadata setting ([#74936](https://github.com/kubernetes/kubernetes/pull/74936), [@rjaini](https://github.com/rjaini))",
-    "author": "rjaini",
-    "author_url": "https://github.com/rjaini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74936",
-    "pr_number": 74936,
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.0"
-  },
-  "74975": {
-    "commit": "bb8ccb6ce72d7648487de21f896cefa2eae0b4ce",
-    "text": "--make-symlinks for hyperkube was marked hidden for a while, This flag is now deprecated and will be removed in a future release.",
-    "markdown": "--make-symlinks for hyperkube was marked hidden for a while, This flag is now deprecated and will be removed in a future release. ([#74975](https://github.com/kubernetes/kubernetes/pull/74975), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74975",
-    "pr_number": 74975,
-    "kinds": ["cleanup"],
-    "sigs": ["release", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "74997": {
-    "commit": "81e8401127db6cc98bb591074d7bd4f2c560fb2a",
-    "text": "New \"dry_run\" metric label (indicating the value of the dryRun query parameter) into the metrics:\n- apiserver_request_total\n- apiserver_request_duration_seconds\nNew \"APPLY\" value for the \"verb\" metric label which indicates a PATCH with \"Content-Type: apply-patch+yaml\". This value is experimental and will only be present if the ServerSideApply alpha feature is enabled.",
-    "markdown": "New \"dry_run\" metric label (indicating the value of the dryRun query parameter) into the metrics:\n  - apiserver_request_total\n  - apiserver_request_duration_seconds\n  New \"APPLY\" value for the \"verb\" metric label which indicates a PATCH with \"Content-Type: apply-patch+yaml\". This value is experimental and will only be present if the ServerSideApply alpha feature is enabled. ([#74997](https://github.com/kubernetes/kubernetes/pull/74997), [@jennybuckley](https://github.com/jennybuckley))\n\n  Courtesy of SIG API Machinery",
-    "author": "jennybuckley",
-    "author_url": "https://github.com/jennybuckley",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74997",
-    "pr_number": 74997,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "74998": {
-    "commit": "e318642946daab9e0330757a3556a1913bb3fc5c",
-    "text": "- Add mechanism for Admission Webhooks to specify which version of AdmissionReview they support\n- Add mechanism for CRD Conversion Webhooks to specify which version of ConversionReview they support",
-    "markdown": "- Add mechanism for Admission Webhooks to specify which version of AdmissionReview they support\n  - Add mechanism for CRD Conversion Webhooks to specify which version of ConversionReview they support ([#74998](https://github.com/kubernetes/kubernetes/pull/74998), [@mbohlool](https://github.com/mbohlool))",
-    "author": "mbohlool",
-    "author_url": "https://github.com/mbohlool",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74998",
-    "pr_number": 74998,
-    "areas": ["apiserver"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.0"
-  },
-  "75003": {
-    "commit": "41b544bcf7034db8e7eadd7189b0a5d9410703e5",
-    "text": "Promote RuntimeClass to beta, and enable by default.",
-    "markdown": "Promote RuntimeClass to beta, and enable by default. ([#75003](https://github.com/kubernetes/kubernetes/pull/75003), [@tallclair](https://github.com/tallclair))\n\n  Courtesy of SIG Auth, SIG Node, and SIG Testing",
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75003",
-    "pr_number": 75003,
-    "kinds": ["feature"],
-    "sigs": ["auth", "node", "testing"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "75037": {
-    "commit": "47063891dd782835170f500a83f37cc98c3c1013",
-    "text": "Fix panic in kubectl cp command",
-    "markdown": "Fix panic in kubectl cp command ([#75037](https://github.com/kubernetes/kubernetes/pull/75037), [@soltysh](https://github.com/soltysh))",
-    "author": "soltysh",
-    "author_url": "https://github.com/soltysh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75037",
-    "pr_number": 75037,
-    "areas": ["kubectl", "security"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.0"
-  },
-  "75082": {
-    "commit": "f0418ac3c2572e22316d6dfa10ab294e49c8cf75",
-    "text": "kubeadm: when calling \"reset\" on a control-plane node, remove the APIEndpoint information for this node from the ClusterStatus in the kubeadm ConfigMap.",
-    "markdown": "kubeadm: when calling \"reset\" on a control-plane node, remove the APIEndpoint information for this node from the ClusterStatus in the kubeadm ConfigMap. ([#75082](https://github.com/kubernetes/kubernetes/pull/75082), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75082",
-    "pr_number": 75082,
-    "areas": ["kubeadm"],
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "75087": {
-    "commit": "43b6ddf3c6b78c1b679f8d4ea88e17bd52005b35",
-    "text": "fix smb unmount issue on Windows",
-    "markdown": "fix smb unmount issue on Windows ([#75087](https://github.com/kubernetes/kubernetes/pull/75087), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75087",
-    "pr_number": 75087,
-    "kinds": ["bug"],
-    "sigs": ["windows"],
-    "release_version": "1.14.0"
-  },
-  "75144": {
-    "commit": "b3ec6c17f137ea0e4662e39a61a9e9e863898a6e",
-    "text": "kubelet: updated logic of verifying a static critical pod.",
-    "markdown": "kubelet: updated logic of verifying a static critical pod. ([#75144](https://github.com/kubernetes/kubernetes/pull/75144), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75144",
-    "pr_number": 75144,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "75149": {
-    "commit": "c534a1b9b305466c907459cfbf8285b6ee70776e",
-    "text": "GCE: bump COS image version to cos-beta-73-11647-64-0",
-    "markdown": "GCE: bump COS image version to cos-beta-73-11647-64-0 ([#75149](https://github.com/kubernetes/kubernetes/pull/75149), [@yguo0905](https://github.com/yguo0905))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "yguo0905",
-    "author_url": "https://github.com/yguo0905",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75149",
-    "pr_number": 75149,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.0"
-  },
-  "75213": {
-    "commit": "8da0c4af29126097f39bfa57067707e76e8923d1",
-    "text": "Reset exponential backoff when storage operation changes",
-    "markdown": "Reset exponential backoff when storage operation changes ([#75213](https://github.com/kubernetes/kubernetes/pull/75213), [@gnufied](https://github.com/gnufied))",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75213",
-    "pr_number": 75213,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.0"
-  },
-  "75256": {
-    "commit": "9cbccd38598e5e2750d39e183aef21a749275087",
-    "text": "Ensure Azure load balancer cleaned up on 404 or 403 when deleting LoadBalancer services.",
-    "markdown": "Ensure Azure load balancer cleaned up on 404 or 403 when deleting LoadBalancer services. ([#75256](https://github.com/kubernetes/kubernetes/pull/75256), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75256",
-    "pr_number": 75256,
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.0"
-  },
-  "75265": {
-    "commit": "aa9cbd112cc34fdba8df25054b6a7a7747c6895d",
-    "text": "UDP Service conntrack entries for ExternalIPs are now correctly cleared when endpoints are added",
-    "markdown": "UDP Service conntrack entries for ExternalIPs are now correctly cleared when endpoints are added ([#75265](https://github.com/kubernetes/kubernetes/pull/75265), [@JacobTanenbaum](https://github.com/JacobTanenbaum))",
-    "author": "JacobTanenbaum",
-    "author_url": "https://github.com/JacobTanenbaum",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75265",
-    "pr_number": 75265,
-    "areas": ["kube-proxy"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.0"
-  },
-  "75279": {
-    "commit": "5229bce04fc7d632f9349480267807a1d41fedf8",
-    "text": "Replace *_admission_latencies_milliseconds_summary and *_admission_latencies_milliseconds metrics due to reporting wrong unit (was labelled milliseconds, but reported seconds), and multiple naming guideline violations (units should be in base units and \"duration\" is the best practice labelling to measure the time a request takes). Please convert to use *_admission_duration_seconds and *_admission_duration_seconds_summary, these now report the unit as described, and follow the instrumentation best practices.",
-    "markdown": "Replace *_admission_latencies_milliseconds_summary and *_admission_latencies_milliseconds metrics due to reporting wrong unit (was labelled milliseconds, but reported seconds), and multiple naming guideline violations (units should be in base units and \"duration\" is the best practice labelling to measure the time a request takes). Please convert to use *_admission_duration_seconds and *_admission_duration_seconds_summary, these now report the unit as described, and follow the instrumentation best practices. ([#75279](https://github.com/kubernetes/kubernetes/pull/75279), [@danielqsj](https://github.com/danielqsj))",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75279",
-    "pr_number": 75279,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "75282": {
-    "commit": "815086f6486eb4e82e5507aaa95f583445221a33",
-    "text": "Allow disable outbound SNAT when Azure standard load balancer is used together with outbound rules.",
-    "markdown": "Allow disable outbound SNAT when Azure standard load balancer is used together with outbound rules. ([#75282](https://github.com/kubernetes/kubernetes/pull/75282), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75282",
-    "pr_number": 75282,
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.0"
-  },
-  "75283": {
-    "commit": "88dc966a8a41fcc694278d7c87114233d2442551",
-    "text": "[IPVS] Allow for transparent kube-proxy restarts",
-    "markdown": "[IPVS] Allow for transparent kube-proxy restarts ([#75283](https://github.com/kubernetes/kubernetes/pull/75283), [@lbernail](https://github.com/lbernail))",
-    "author": "lbernail",
-    "author_url": "https://github.com/lbernail",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75283",
-    "pr_number": 75283,
-    "areas": ["ipvs"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.0"
-  },
-  "75295": {
-    "commit": "59140d647449a41ecbe465fec3ec7018f85c6cff",
-    "text": "[IPVS] Introduces flag ipvs-strict-arp to configure stricter ARP sysctls, defaulting to false to preserve existing behaviors. This was enabled by default in 1.13.0, which impacted a few CNI plugins.",
-    "markdown": "[IPVS] Introduces flag ipvs-strict-arp to configure stricter ARP sysctls, defaulting to false to preserve existing behaviors. This was enabled by default in 1.13.0, which impacted a few CNI plugins. ([#75295](https://github.com/kubernetes/kubernetes/pull/75295), [@lbernail](https://github.com/lbernail))",
-    "author": "lbernail",
-    "author_url": "https://github.com/lbernail",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75295",
-    "pr_number": 75295,
-    "areas": ["ipvs"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.0"
-  },
-  "75367": {
-    "commit": "0f6196878d585a18f02e879e56f75bc546f21a20",
-    "text": "Fix dockershim panic issues when deleting docker images.",
-    "markdown": "Fix dockershim panic issues when deleting docker images. ([#75367](https://github.com/kubernetes/kubernetes/pull/75367), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75367",
-    "pr_number": 75367,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.0"
-  },
-  "75371": {
-    "commit": "a4f2590bd8400ca43e2cf09882f493611a3ce289",
-    "text": "fix race condition issue for smb mount on windows",
-    "markdown": "fix race condition issue for smb mount on windows ([#75371](https://github.com/kubernetes/kubernetes/pull/75371), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75371",
-    "pr_number": 75371,
-    "kinds": ["bug"],
-    "sigs": ["storage", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "75376": {
-    "commit": "ac6b1a1fc514e2b32c080b0de5da8c627b42b279",
-    "text": "Fix some potential deadlocks and file descriptor leaking for inotify watches.",
-    "markdown": "Fix some potential deadlocks and file descriptor leaking for inotify watches. ([#75376](https://github.com/kubernetes/kubernetes/pull/75376), [@cpuguy83](https://github.com/cpuguy83))",
-    "author": "cpuguy83",
-    "author_url": "https://github.com/cpuguy83",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75376",
-    "pr_number": 75376,
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.0"
-  },
-  "75391": {
-    "commit": "0687949f5e1eba72aea933ceb9fa967736306c1f",
-    "text": "Update CSI version to 1.1",
-    "markdown": "Update CSI version to 1.1 ([#75391](https://github.com/kubernetes/kubernetes/pull/75391), [@gnufied](https://github.com/gnufied))",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75391",
-    "pr_number": 75391,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.0"
-  },
-  "75413": {
-    "commit": "f3fdac12a0e1776989318d895d4d9a2d01015e7d",
-    "text": "Update to use go1.12.1",
-    "markdown": "Update to use go1.12.1 ([#75413](https://github.com/kubernetes/kubernetes/pull/75413), [@BenTheElder](https://github.com/BenTheElder))",
-    "author": "BenTheElder",
-    "author_url": "https://github.com/BenTheElder",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75413",
-    "pr_number": 75413,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["release", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "75415": {
-    "commit": "4d120473fd1f73019d3e0950f3ae2825fbd4d777",
-    "text": "kubeadm: Allow certain certs/keys to be missing on the secret when transferring secrets using `--experimental-upload-certs` feature",
-    "markdown": "kubeadm: Allow certain certs/keys to be missing on the secret when transferring secrets using `--experimental-upload-certs` feature ([#75415](https://github.com/kubernetes/kubernetes/pull/75415), [@ereslibre](https://github.com/ereslibre))",
-    "author": "ereslibre",
-    "author_url": "https://github.com/ereslibre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75415",
-    "pr_number": 75415,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "75416": {
-    "commit": "3512757882bed33c5652ef7a6b970806c0aac40b",
-    "text": "Health check (liveness \u0026 readiness) probes using an HTTPGetAction will no longer follow redirects to different hostnames from the original probe request. Instead, these non-local redirects will be treated as a Success (the documented behavior). In this case an event with reason \"ProbeWarning\" will be generated, indicating that the redirect was ignored. If you were previously relying on the redirect to run health checks against different endpoints, you will need to perform the healthcheck logic outside the Kubelet, for instance by proxying the external endpoint rather than redirecting to it.",
-    "markdown": "Health check (liveness \u0026 readiness) probes using an HTTPGetAction will no longer follow redirects to different hostnames from the original probe request. Instead, these non-local redirects will be treated as a Success (the documented behavior). In this case an event with reason \"ProbeWarning\" will be generated, indicating that the redirect was ignored. If you were previously relying on the redirect to run health checks against different endpoints, you will need to perform the healthcheck logic outside the Kubelet, for instance by proxying the external endpoint rather than redirecting to it. ([#75416](https://github.com/kubernetes/kubernetes/pull/75416), [@tallclair](https://github.com/tallclair))\n\n  Courtesy of SIG Network, SIG Node, and SIG Testing",
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75416",
-    "pr_number": 75416,
-    "areas": ["kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["network", "node", "testing"],
-    "action_required": true,
-    "release_version": "1.14.0"
-  },
-  "75420": {
-    "commit": "b61baea13594f8cacd8e7db86aa68cbd967b9a5e",
-    "text": "kubeadm init output for cluster WITH `controlPlaneEndpoint`, but created WITHOUT `--upload-certs`",
-    "markdown": "kubeadm init output for cluster WITH `controlPlaneEndpoint`, but created WITHOUT `--upload-certs` ([#75420](https://github.com/kubernetes/kubernetes/pull/75420), [@fabriziopandini](https://github.com/fabriziopandini))",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75420",
-    "pr_number": 75420,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  },
-  "75451": {
-    "commit": "1d441c1f93e7cc44f8a200df28a2c8a4bee3a2bb",
-    "text": "Restores --username and --password flags to kubectl",
-    "markdown": "Restores --username and --password flags to kubectl ([#75451](https://github.com/kubernetes/kubernetes/pull/75451), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75451",
-    "pr_number": 75451,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["auth", "cli"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "75480": {
-    "commit": "164fad5e3f447e3eb55154cc58bcbe4b3016d129",
-    "text": "Update Cluster Autoscaler to 1.14.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.14.0",
-    "markdown": "Update Cluster Autoscaler to 1.14.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.14.0 ([#75480](https://github.com/kubernetes/kubernetes/pull/75480), [@losipiuk](https://github.com/losipiuk))",
-    "author": "losipiuk",
-    "author_url": "https://github.com/losipiuk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75480",
-    "pr_number": 75480,
-    "kinds": ["bug"],
-    "sigs": ["autoscaling", "cluster-lifecycle"],
-    "duplicate": true,
-    "release_version": "1.14.0"
-  },
-  "75487": {
-    "commit": "e6272b887b81a62e6f06b7fac4b3b61d1c8bf657",
-    "text": "kubeadm: fix the machine readability of \"kubeadm token create --print-join-command\"",
-    "markdown": "kubeadm: fix the machine readability of \"kubeadm token create --print-join-command\" ([#75487](https://github.com/kubernetes/kubernetes/pull/75487), [@displague](https://github.com/displague))",
-    "author": "displague",
-    "author_url": "https://github.com/displague",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75487",
-    "pr_number": 75487,
-    "areas": ["kubeadm"],
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.0"
-  }
-}
diff --git a/src/assets/release-notes-1.14.1.json b/src/assets/release-notes-1.14.1.json
deleted file mode 100644
index b1640eb2..00000000
--- a/src/assets/release-notes-1.14.1.json
+++ /dev/null
@@ -1,570 +0,0 @@
-{
-  "66635": {
-    "commit": "4e397d971a72f5bc6dbffbacca7297f17e17c572",
-    "text": "The `system:aws-cloud-provider` cluster role, deprecated in v1.13, is no longer auto-created. Deployments using the AWS cloud provider should grant required permissions to the `aws-cloud-provider` service account in the `kube-system` namespace as part of deployment.",
-    "markdown": "The `system:aws-cloud-provider` cluster role, deprecated in v1.13, is no longer auto-created. Deployments using the AWS cloud provider should grant required permissions to the `aws-cloud-provider` service account in the `kube-system` namespace as part of deployment. ([#66635](https://github.com/kubernetes/kubernetes/pull/66635), [@wgliang](https://github.com/wgliang))",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/66635",
-    "pr_number": 66635,
-    "sigs": ["auth", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.14.1"
-  },
-  "72179": {
-    "commit": "11d472ea01f23063158e1f0e4fba37144e4edc59",
-    "text": "Transition service account controller clients to TokenRequest API",
-    "markdown": "Transition service account controller clients to TokenRequest API ([#72179](https://github.com/kubernetes/kubernetes/pull/72179), [@WanLinghao](https://github.com/WanLinghao))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, and SIG Testing",
-    "author": "WanLinghao",
-    "author_url": "https://github.com/WanLinghao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72179",
-    "pr_number": 72179,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "apps", "auth", "testing"],
-    "feature": true,
-    "release_version": "1.14.1"
-  },
-  "72384": {
-    "commit": "b9dbb458fb919154620fc2525f56dc515b44e68e",
-    "text": "Object count quota is now supported for namespaced custom resources using the `count/\u003cresource\u003e.\u003cgroup\u003e` syntax.",
-    "markdown": "Object count quota is now supported for namespaced custom resources using the `count/\u003cresource\u003e.\u003cgroup\u003e` syntax. ([#72384](https://github.com/kubernetes/kubernetes/pull/72384), [@zhouhaibing089](https://github.com/zhouhaibing089))\n\n  Courtesy of SIG API Machinery, SIG Apps, and SIG Testing",
-    "author": "zhouhaibing089",
-    "author_url": "https://github.com/zhouhaibing089",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72384",
-    "pr_number": 72384,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "apps", "testing"],
-    "feature": true,
-    "release_version": "1.14.1"
-  },
-  "72454": {
-    "commit": "5a6c66a729b4fa23d8fd254dbb0098396a192227",
-    "text": "In case kubeadm can't access the current Kubernetes version remotely and fails to parse\nthe git-based version it falls back to a static predefined value of\nk8s.io/kubernetes/cmd/kubeadm/app/constants.CurrentKubernetesVersion.",
-    "markdown": "In case kubeadm can't access the current Kubernetes version remotely and fails to parse\n  the git-based version it falls back to a static predefined value of\n  k8s.io/kubernetes/cmd/kubeadm/app/constants.CurrentKubernetesVersion. ([#72454](https://github.com/kubernetes/kubernetes/pull/72454), [@rojkov](https://github.com/rojkov))",
-    "author": "rojkov",
-    "author_url": "https://github.com/rojkov",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72454",
-    "pr_number": 72454,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup", "failing-test"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.1"
-  },
-  "73863": {
-    "commit": "76b1d8351720c5a889536e5db9de972d40ed43a2",
-    "text": "Count PVCs that are unbound towards attach limit",
-    "markdown": "Count PVCs that are unbound towards attach limit ([#73863](https://github.com/kubernetes/kubernetes/pull/73863), [@gnufied](https://github.com/gnufied))",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73863",
-    "pr_number": 73863,
-    "kinds": ["bug"],
-    "sigs": ["scheduling", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.1"
-  },
-  "74159": {
-    "commit": "7d15d41eb87f8e6977a8947efe3edf7a75c1d01d",
-    "text": "Support parsing more v1.Taint forms. `key:effect`, `key=:effect-` are now accepted.",
-    "markdown": "Support parsing more v1.Taint forms. `key:effect`, `key=:effect-` are now accepted. ([#74159](https://github.com/kubernetes/kubernetes/pull/74159), [@dlipovetsky](https://github.com/dlipovetsky))",
-    "author": "dlipovetsky",
-    "author_url": "https://github.com/dlipovetsky",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74159",
-    "pr_number": 74159,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.14.1"
-  },
-  "74321": {
-    "commit": "ec64aef25f2fc8c7f3093d16566e6d2e40b61e4e",
-    "text": "watch can now be enabled  for events using the flag --watch-cache-sizes on kube-apiserver",
-    "markdown": "watch can now be enabled  for events using the flag --watch-cache-sizes on kube-apiserver ([#74321](https://github.com/kubernetes/kubernetes/pull/74321), [@yastij](https://github.com/yastij))\n\n  Courtesy of SIG API Machinery",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74321",
-    "pr_number": 74321,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.1"
-  },
-  "74747": {
-    "commit": "a8cbb2250624524c8f3bba33f0ec09e8a252c26b",
-    "text": "- Fixed a potential deadlock in resource quota controller\n- Enabled recording partial usage info for quota objects specifying multiple resources, when only some of the resources' usage can be determined.",
-    "markdown": "- Fixed a potential deadlock in resource quota controller\n  - Enabled recording partial usage info for quota objects specifying multiple resources, when only some of the resources' usage can be determined. ([#74747](https://github.com/kubernetes/kubernetes/pull/74747), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74747",
-    "pr_number": 74747,
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery", "apps"],
-    "duplicate": true,
-    "release_version": "1.14.1"
-  },
-  "74877": {
-    "commit": "d920430ce4a9140102d06664c6aa1cd6792cab05",
-    "text": "`k8s.io/kubernetes` and published components (like `k8s.io/client-go` and `k8s.io/api`) now publish go module files containing dependency version information. See http://git.k8s.io/client-go/INSTALL.md\u0026#35;go-modules for details on consuming `k8s.io/client-go` using go modules.",
-    "markdown": "`k8s.io/kubernetes` and published components (like `k8s.io/client-go` and `k8s.io/api`) now publish go module files containing dependency version information. See http://git.k8s.io/client-go/INSTALL.md\u0026#35;go-modules for details on consuming `k8s.io/client-go` using go modules. ([#74877](https://github.com/kubernetes/kubernetes/pull/74877), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery, SIG CLI, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Node, and SIG Release",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74877",
-    "pr_number": 74877,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "release-eng"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "node", "release"],
-    "feature": true,
-    "release_version": "1.14.1"
-  },
-  "74889": {
-    "commit": "657951c3ed79ac43191ed0ef1368a61795b1a572",
-    "text": "Users may now execute `get-kube-binaries.sh` to request a client for an OS/Arch unlike the one of the host on which the script is invoked.",
-    "markdown": "Users may now execute `get-kube-binaries.sh` to request a client for an OS/Arch unlike the one of the host on which the script is invoked. ([#74889](https://github.com/kubernetes/kubernetes/pull/74889), [@akutz](https://github.com/akutz))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "akutz",
-    "author_url": "https://github.com/akutz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74889",
-    "pr_number": 74889,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.1"
-  },
-  "75062": {
-    "commit": "055061637a465fd2333431f2361c9f915bdf951d",
-    "text": "Allow to read OpenStack user credentials from a secret instead of a local config file.",
-    "markdown": "Allow to read OpenStack user credentials from a secret instead of a local config file. ([#75062](https://github.com/kubernetes/kubernetes/pull/75062), [@Fedosin](https://github.com/Fedosin))\n\n  Courtesy of SIG Cloud Provider, and SIG Storage",
-    "author": "Fedosin",
-    "author_url": "https://github.com/Fedosin",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75062",
-    "pr_number": 75062,
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider", "storage"],
-    "feature": true,
-    "release_version": "1.14.1"
-  },
-  "75072": {
-    "commit": "0118630abf4f47d17770f08e7aad862107d2ce27",
-    "text": "Add name validation for dynamic client methods in client-go",
-    "markdown": "Add name validation for dynamic client methods in client-go ([#75072](https://github.com/kubernetes/kubernetes/pull/75072), [@lblackstone](https://github.com/lblackstone))",
-    "author": "lblackstone",
-    "author_url": "https://github.com/lblackstone",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75072",
-    "pr_number": 75072,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.1"
-  },
-  "75179": {
-    "commit": "9c53f24012a0c05662a1b5626cce0271bbe881e4",
-    "text": "kubeadm: Support for deprecated old kubeadm v1alpha3 config is totally removed.",
-    "markdown": "kubeadm: Support for deprecated old kubeadm v1alpha3 config is totally removed. ([#75179](https://github.com/kubernetes/kubernetes/pull/75179), [@rosti](https://github.com/rosti))",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75179",
-    "pr_number": 75179,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.1"
-  },
-  "75362": {
-    "commit": "da018a6bfa2214c7068644ba48d25939e44b9db5",
-    "text": "[stackdriver addon] Bump prometheus-to-sd to v0.5.0 to pick up security fixes.\n[fluentd-gcp addon] Bump fluentd-gcp-scaler to v0.5.1 to pick up security fixes.\n[fluentd-gcp addon] Bump event-exporter to v0.2.4 to pick up security fixes.\n[fluentd-gcp addon] Bump prometheus-to-sd to v0.5.0 to pick up security fixes.\n[metatada-proxy addon] Bump prometheus-to-sd v0.5.0 to pick up security fixes.",
-    "markdown": "[stackdriver addon] Bump prometheus-to-sd to v0.5.0 to pick up security fixes.\n  [fluentd-gcp addon] Bump fluentd-gcp-scaler to v0.5.1 to pick up security fixes.\n  [fluentd-gcp addon] Bump event-exporter to v0.2.4 to pick up security fixes.\n  [fluentd-gcp addon] Bump prometheus-to-sd to v0.5.0 to pick up security fixes.\n  [metatada-proxy addon] Bump prometheus-to-sd v0.5.0 to pick up security fixes. ([#75362](https://github.com/kubernetes/kubernetes/pull/75362), [@serathius](https://github.com/serathius))",
-    "author": "serathius",
-    "author_url": "https://github.com/serathius",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75362",
-    "pr_number": 75362,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.1"
-  },
-  "75389": {
-    "commit": "c79fbabf234bea36f7b870da8e763c542c804be0",
-    "text": "Paginate requests from the kube-apiserver watch cache to etcd in chunks.\nPaginate reflector init and resync List calls that are not served by watch cache.",
-    "markdown": "Paginate requests from the kube-apiserver watch cache to etcd in chunks.\n  Paginate reflector init and resync List calls that are not served by watch cache. ([#75389](https://github.com/kubernetes/kubernetes/pull/75389), [@jpbetz](https://github.com/jpbetz))\n\n  Courtesy of SIG API Machinery, SIG Apps, and SIG Testing",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75389",
-    "pr_number": 75389,
-    "areas": ["apiserver", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "apps", "testing"],
-    "feature": true,
-    "release_version": "1.14.1"
-  },
-  "75483": {
-    "commit": "483cd0db09e9af9e7a37f59ed90c33264eee0740",
-    "text": "In the 'kubectl describe' output, the fields with names containing special characters are displayed as-is without any pretty formatting.",
-    "markdown": "In the 'kubectl describe' output, the fields with names containing special characters are displayed as-is without any pretty formatting. ([#75483](https://github.com/kubernetes/kubernetes/pull/75483), [@gsadhani](https://github.com/gsadhani))",
-    "author": "gsadhani",
-    "author_url": "https://github.com/gsadhani",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75483",
-    "pr_number": 75483,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.1"
-  },
-  "75515": {
-    "commit": "9c973c6d2c33e88521ebcebec2fdd9cbddccd857",
-    "text": "Support vSphere SAML token auth when using Zones",
-    "markdown": "Support vSphere SAML token auth when using Zones ([#75515](https://github.com/kubernetes/kubernetes/pull/75515), [@dougm](https://github.com/dougm))",
-    "author": "dougm",
-    "author_url": "https://github.com/dougm",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75515",
-    "pr_number": 75515,
-    "areas": ["cloudprovider", "provider/vmware"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.1"
-  },
-  "75531": {
-    "commit": "4b3eb600811339fe1b3972672edafa1fe10309ec",
-    "text": "CRI API will now be available in the kubernetes/cri-api repository",
-    "markdown": "CRI API will now be available in the kubernetes/cri-api repository ([#75531](https://github.com/kubernetes/kubernetes/pull/75531), [@dims](https://github.com/dims))\n\n  Courtesy of SIG Network, SIG Node, SIG Release, and SIG Testing",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75531",
-    "pr_number": 75531,
-    "areas": ["kubelet", "release-eng", "test"],
-    "kinds": ["feature"],
-    "sigs": ["network", "node", "release", "testing"],
-    "feature": true,
-    "release_version": "1.14.1"
-  },
-  "75580": {
-    "commit": "2fa5be16334dfaca05049798f2cb73ac7979a4b6",
-    "text": "Support specify the Resource Group of Route Table when update Pod network route (Azure)",
-    "markdown": "Support specify the Resource Group of Route Table when update Pod network route (Azure) ([#75580](https://github.com/kubernetes/kubernetes/pull/75580), [@suker200](https://github.com/suker200))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "suker200",
-    "author_url": "https://github.com/suker200",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75580",
-    "pr_number": 75580,
-    "areas": ["cloudprovider"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.14.1"
-  },
-  "75587": {
-    "commit": "dabeb20a1ae379a27359ceb1bc77ff49dbd6db12",
-    "text": "The AWS credential provider can now obtain ECR credentials even without the AWS cloud provider or being on an EC2 instance. Additionally, AWS credential provider caching has been improved to honor the ECR credential timeout.",
-    "markdown": "The AWS credential provider can now obtain ECR credentials even without the AWS cloud provider or being on an EC2 instance. Additionally, AWS credential provider caching has been improved to honor the ECR credential timeout. ([#75587](https://github.com/kubernetes/kubernetes/pull/75587), [@tiffanyfay](https://github.com/tiffanyfay))",
-    "author": "tiffanyfay",
-    "author_url": "https://github.com/tiffanyfay",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75587",
-    "pr_number": 75587,
-    "areas": ["cloudprovider", "kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["cloud-provider", "node"],
-    "duplicate": true,
-    "release_version": "1.14.1"
-  },
-  "75602": {
-    "commit": "3312da83c9310e9b2b54aa2556f9da06b4bb1394",
-    "text": "The Kubelet now properly requests protobuf objects where they are\nsupported from the apiserver, reducing load in large clusters.",
-    "markdown": "The Kubelet now properly requests protobuf objects where they are\n  supported from the apiserver, reducing load in large clusters. ([#75602](https://github.com/kubernetes/kubernetes/pull/75602), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75602",
-    "pr_number": 75602,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "node", "scalability"],
-    "duplicate": true,
-    "release_version": "1.14.1"
-  },
-  "75622": {
-    "commit": "eee8af93baf00786d129d233d0e80a5e01150808",
-    "text": "StatefulSet controllers no longer force a resync every 30 seconds when nothing has changed.",
-    "markdown": "StatefulSet controllers no longer force a resync every 30 seconds when nothing has changed. ([#75622](https://github.com/kubernetes/kubernetes/pull/75622), [@anyasabo](https://github.com/anyasabo))",
-    "author": "anyasabo",
-    "author_url": "https://github.com/anyasabo",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75622",
-    "pr_number": 75622,
-    "kinds": ["cleanup"],
-    "sigs": ["apps"],
-    "release_version": "1.14.1"
-  },
-  "75646": {
-    "commit": "e6d274233c2a4d02fc927d8a62b44b8b74811382",
-    "text": "kubeadm: implement deletion of multiple bootstrap tokens at once",
-    "markdown": "kubeadm: implement deletion of multiple bootstrap tokens at once ([#75646](https://github.com/kubernetes/kubernetes/pull/75646), [@bart0sh](https://github.com/bart0sh))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "bart0sh",
-    "author_url": "https://github.com/bart0sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75646",
-    "pr_number": 75646,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.1"
-  },
-  "75712": {
-    "commit": "16a03518812fa29c87560edd0fea26d299c3d8e2",
-    "text": "Add completed job status in Cronjob event.",
-    "markdown": "Add completed job status in Cronjob event. ([#75712](https://github.com/kubernetes/kubernetes/pull/75712), [@danielqsj](https://github.com/danielqsj))\n\n  Courtesy of SIG Apps",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75712",
-    "pr_number": 75712,
-    "kinds": ["feature"],
-    "sigs": ["apps"],
-    "feature": true,
-    "release_version": "1.14.1"
-  },
-  "75747": {
-    "commit": "42877480ad9b784956c5d9228ca6e2232778ce5f",
-    "text": "CSI alpha CRDs have been removed",
-    "markdown": "CSI alpha CRDs have been removed ([#75747](https://github.com/kubernetes/kubernetes/pull/75747), [@msau42](https://github.com/msau42))",
-    "author": "msau42",
-    "author_url": "https://github.com/msau42",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75747",
-    "pr_number": 75747,
-    "areas": ["release-eng", "test"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["apps", "cluster-lifecycle", "release", "storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.1"
-  },
-  "75750": {
-    "commit": "f25fa0e9e1ce2e275d2a9f468391783b0f04e56e",
-    "text": "Adds a new \"storage_operation_status_count\" metric for kube-controller-manager and kubelet to count success and error statues.",
-    "markdown": "Adds a new \"storage_operation_status_count\" metric for kube-controller-manager and kubelet to count success and error statues. ([#75750](https://github.com/kubernetes/kubernetes/pull/75750), [@msau42](https://github.com/msau42))",
-    "author": "msau42",
-    "author_url": "https://github.com/msau42",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75750",
-    "pr_number": 75750,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["instrumentation", "storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.1"
-  },
-  "75771": {
-    "commit": "fc556f0d9ba59ac47c76e4d60a9ba261e0c622f3",
-    "text": "client-go: The `rest.AnonymousClientConfig(*rest.Config) *rest.Config` helper method no longer copies custom `Transport` and `WrapTransport` fields, because those can be used to inject user credentials.",
-    "markdown": "client-go: The `rest.AnonymousClientConfig(*rest.Config) *rest.Config` helper method no longer copies custom `Transport` and `WrapTransport` fields, because those can be used to inject user credentials. ([#75771](https://github.com/kubernetes/kubernetes/pull/75771), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75771",
-    "pr_number": 75771,
-    "areas": ["client-libraries"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth"],
-    "duplicate": true,
-    "release_version": "1.14.1"
-  },
-  "75781": {
-    "commit": "a0e71b49b6b3eb1ffba097b0c12edd4029fd3363",
-    "text": "Increased verbose level for local openapi aggregation logs to avoid flooding the log during normal operation",
-    "markdown": "Increased verbose level for local openapi aggregation logs to avoid flooding the log during normal operation ([#75781](https://github.com/kubernetes/kubernetes/pull/75781), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75781",
-    "pr_number": 75781,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.1"
-  },
-  "75799": {
-    "commit": "861f1fcb2e12c0a7eba5a8e8c894cbed7076ee11",
-    "text": "The Node.Status.Volumes.Attached.DevicePath field is now unset for CSI volumes. Update any external controllers that depend on this field.",
-    "markdown": "The Node.Status.Volumes.Attached.DevicePath field is now unset for CSI volumes. Update any external controllers that depend on this field. ([#75799](https://github.com/kubernetes/kubernetes/pull/75799), [@msau42](https://github.com/msau42))\n\n  Courtesy of SIG Storage",
-    "author": "msau42",
-    "author_url": "https://github.com/msau42",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75799",
-    "pr_number": 75799,
-    "kinds": ["cleanup"],
-    "sigs": ["storage"],
-    "action_required": true,
-    "release_version": "1.14.1"
-  },
-  "75847": {
-    "commit": "91f40aec8ad5b992d8057b4fab0fa287ee6bdee9",
-    "text": "kubeadm: preflight checks on external etcd certificates are now skipped when joining a control-plane node with automatic copy of cluster certificates (--certificate-key)",
-    "markdown": "kubeadm: preflight checks on external etcd certificates are now skipped when joining a control-plane node with automatic copy of cluster certificates (--certificate-key) ([#75847](https://github.com/kubernetes/kubernetes/pull/75847), [@fabriziopandini](https://github.com/fabriziopandini))",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75847",
-    "pr_number": 75847,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.1"
-  },
-  "75855": {
-    "commit": "6fef6e8613c0b4fec54b129e9c8b74daff253296",
-    "text": "GCE Windows nodes will rely solely on kubernetes and kube-proxy (and not the GCE agent) for network address management.",
-    "markdown": "GCE Windows nodes will rely solely on kubernetes and kube-proxy (and not the GCE agent) for network address management. ([#75855](https://github.com/kubernetes/kubernetes/pull/75855), [@pjh](https://github.com/pjh))",
-    "author": "pjh",
-    "author_url": "https://github.com/pjh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75855",
-    "pr_number": 75855,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.1"
-  },
-  "75857": {
-    "commit": "2792f1a24e1d9e6aaccde3661cc6a25c5d728d88",
-    "text": "Support both JSON and YAML for scheduler configuration.",
-    "markdown": "Support both JSON and YAML for scheduler configuration. ([#75857](https://github.com/kubernetes/kubernetes/pull/75857), [@danielqsj](https://github.com/danielqsj))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75857",
-    "pr_number": 75857,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.1"
-  },
-  "75866": {
-    "commit": "2c675818ed46579ca91ed88a4222cc87c7328a16",
-    "text": "Add --image-repository flag to \"kubeadm config images\".",
-    "markdown": "Add --image-repository flag to \"kubeadm config images\". ([#75866](https://github.com/kubernetes/kubernetes/pull/75866), [@jmkeyes](https://github.com/jmkeyes))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "jmkeyes",
-    "author_url": "https://github.com/jmkeyes",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75866",
-    "pr_number": 75866,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.1"
-  },
-  "75887": {
-    "commit": "6681c127d308f4bb4ab92435427e7f51d6c34f67",
-    "text": "Fixes a regression proxying responses from aggregated API servers which could cause watch requests to hang until the first event was received",
-    "markdown": "Fixes a regression proxying responses from aggregated API servers which could cause watch requests to hang until the first event was received ([#75887](https://github.com/kubernetes/kubernetes/pull/75887), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75887",
-    "pr_number": 75887,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.1"
-  },
-  "75900": {
-    "commit": "8f304510c5345cc0c5132c5219304583a595b139",
-    "text": "kubeadm: fix \"upgrade plan\" not defaulting to a \"stable\" version if no version argument is passed",
-    "markdown": "kubeadm: fix \"upgrade plan\" not defaulting to a \"stable\" version if no version argument is passed ([#75900](https://github.com/kubernetes/kubernetes/pull/75900), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75900",
-    "pr_number": 75900,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.1"
-  },
-  "75969": {
-    "commit": "c3cc31797a4934b29fc1244a5828ae02e3c4b68d",
-    "text": "`StubDomains` and `Upstreamnameserver` which contains a service name will be omitted while translating to the equivalent CoreDNS config.",
-    "markdown": "`StubDomains` and `Upstreamnameserver` which contains a service name will be omitted while translating to the equivalent CoreDNS config. ([#75969](https://github.com/kubernetes/kubernetes/pull/75969), [@rajansandeep](https://github.com/rajansandeep))",
-    "author": "rajansandeep",
-    "author_url": "https://github.com/rajansandeep",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75969",
-    "pr_number": 75969,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.1"
-  },
-  "75971": {
-    "commit": "a3ae6c43c266ebfb7496430f2938993c6de81440",
-    "text": "Resource list requests for PartialObjectMetadata now correctly return list metadata like the resourceVersion and the continue token.",
-    "markdown": "Resource list requests for PartialObjectMetadata now correctly return list metadata like the resourceVersion and the continue token. ([#75971](https://github.com/kubernetes/kubernetes/pull/75971), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75971",
-    "pr_number": 75971,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.1"
-  },
-  "75985": {
-    "commit": "16db83b257af11370b575c362e79ee5d7f1b09ff",
-    "text": "Tolerations with the same key and effect will be merged into one which has the value of the latest toleration for best effort pods.",
-    "markdown": "Tolerations with the same key and effect will be merged into one which has the value of the latest toleration for best effort pods. ([#75985](https://github.com/kubernetes/kubernetes/pull/75985), [@ravisantoshgudimetla](https://github.com/ravisantoshgudimetla))",
-    "author": "ravisantoshgudimetla",
-    "author_url": "https://github.com/ravisantoshgudimetla",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75985",
-    "pr_number": 75985,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.1"
-  },
-  "76039": {
-    "commit": "41691a031a5dd1b8e8912061dcb5efe488167115",
-    "text": "Ensures the conformance test image saves results before exiting when ginkgo returns non-zero value.",
-    "markdown": "Ensures the conformance test image saves results before exiting when ginkgo returns non-zero value. ([#76039](https://github.com/kubernetes/kubernetes/pull/76039), [@johnSchnake](https://github.com/johnSchnake))",
-    "author": "johnSchnake",
-    "author_url": "https://github.com/johnSchnake",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76039",
-    "pr_number": 76039,
-    "areas": ["conformance"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "release", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.1"
-  },
-  "76063": {
-    "commit": "658c290ed507df073e8a0729471d40676e752aa9",
-    "text": "give users the option to suppress detailed output in integration test",
-    "markdown": "give users the option to suppress detailed output in integration test ([#76063](https://github.com/kubernetes/kubernetes/pull/76063), [@Huang-Wei](https://github.com/Huang-Wei))\n\n  Courtesy of SIG Testing",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76063",
-    "pr_number": 76063,
-    "kinds": ["feature"],
-    "sigs": ["testing"],
-    "feature": true,
-    "release_version": "1.14.1"
-  },
-  "76099": {
-    "commit": "3e7134891f9e9fc4afe3fe28e0fb11721afbe945",
-    "text": "GCE/Windows: disable stackdriver logging agent to prevent node startup failures",
-    "markdown": "GCE/Windows: disable stackdriver logging agent to prevent node startup failures ([#76099](https://github.com/kubernetes/kubernetes/pull/76099), [@yujuhong](https://github.com/yujuhong))",
-    "author": "yujuhong",
-    "author_url": "https://github.com/yujuhong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76099",
-    "pr_number": 76099,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.1"
-  },
-  "76109": {
-    "commit": "71f4c9a51c60a38f71dae7ab189bf25679accfbb",
-    "text": "kube-proxy no longer automatically cleans up network rules created by running kube-proxy in other modes. If you are switching the mode that kube-proxy is in running in (EG: iptables to IPVS), you will need to run `kube-proxy --cleanup`, or restart the worker node (recommended) before restarting kube-proxy.\n\nIf you are not switching kube-proxy between different modes, this change should not require any action.",
-    "markdown": "kube-proxy no longer automatically cleans up network rules created by running kube-proxy in other modes. If you are switching the mode that kube-proxy is in running in (EG: iptables to IPVS), you will need to run `kube-proxy --cleanup`, or restart the worker node (recommended) before restarting kube-proxy.\n  \n  If you are not switching kube-proxy between different modes, this change should not require any action. ([#76109](https://github.com/kubernetes/kubernetes/pull/76109), [@vllry](https://github.com/vllry))",
-    "author": "vllry",
-    "author_url": "https://github.com/vllry",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76109",
-    "pr_number": 76109,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.1"
-  },
-  "76211": {
-    "commit": "3e954d3bd30c5293ecc4cfd7f259852875ea7f79",
-    "text": "Fixes a NPD bug on GCI, so that it disables glog writing to files for log-counter",
-    "markdown": "Fixes a NPD bug on GCI, so that it disables glog writing to files for log-counter ([#76211](https://github.com/kubernetes/kubernetes/pull/76211), [@wangzhen127](https://github.com/wangzhen127))",
-    "author": "wangzhen127",
-    "author_url": "https://github.com/wangzhen127",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76211",
-    "pr_number": 76211,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "node"],
-    "duplicate": true,
-    "release_version": "1.14.1"
-  }
-}
diff --git a/src/assets/release-notes-1.14.10.json b/src/assets/release-notes-1.14.10.json
deleted file mode 100644
index 4037543a..00000000
--- a/src/assets/release-notes-1.14.10.json
+++ /dev/null
@@ -1,1824 +0,0 @@
-{
-  "72046": {
-    "commit": "d9be37e926b97c1687abef74953df4844f013972",
-    "text": "Support Service Topology",
-    "markdown": "Support Service Topology ([#72046](https://github.com/kubernetes/kubernetes/pull/72046), [@m1093782566](https://github.com/m1093782566))\n\n  Courtesy of SIG Apps, SIG Auth, and SIG Network",
-    "author": "m1093782566",
-    "author_url": "https://github.com/m1093782566",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72046",
-    "pr_number": 72046,
-    "areas": ["ipvs"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["apps", "auth", "network"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "74026": {
-    "commit": "cb2684c41671f78563f8e92b6c5fafbf45471534",
-    "text": "All nodes need to be drained before upgrading Kubernetes cluster, because paths used for block volumes are changed in this release, so on-line upgrade of nodes aren't allowed.",
-    "markdown": "All nodes need to be drained before upgrading Kubernetes cluster, because paths used for block volumes are changed in this release, so on-line upgrade of nodes aren't allowed. ([#74026](https://github.com/kubernetes/kubernetes/pull/74026), [@mkimuram](https://github.com/mkimuram))",
-    "author": "mkimuram",
-    "author_url": "https://github.com/mkimuram",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74026",
-    "pr_number": 74026,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "74525": {
-    "commit": "928817a26a84d9e3076d110ea30ba994912aa477",
-    "text": "TBA",
-    "markdown": "TBA ([#74525](https://github.com/kubernetes/kubernetes/pull/74525), [@gliush](https://github.com/gliush))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Autoscaling, and SIG CLI",
-    "author": "gliush",
-    "author_url": "https://github.com/gliush",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74525",
-    "pr_number": 74525,
-    "areas": ["kubectl"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "autoscaling", "cli"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "76310": {
-    "commit": "bb55aa7c54c327fc0d71671e978e7626ce366497",
-    "text": "Critical pods can now be created in namespaces other than kube-system. To limit critical pods to the kube-system namespace, cluster admins should create an admission configuration file limiting critical pods by default, and a matching quota object in the `kube-system` namespace permitting critical pods in that namespace. See https://kubernetes.io/docs/concepts/policy/resource-quotas/\u0026#35;limit-priority-class-consumption-by-default for details.",
-    "markdown": "Critical pods can now be created in namespaces other than kube-system. To limit critical pods to the kube-system namespace, cluster admins should create an admission configuration file limiting critical pods by default, and a matching quota object in the `kube-system` namespace permitting critical pods in that namespace. See https://kubernetes.io/docs/concepts/policy/resource-quotas/\u0026#35;limit-priority-class-consumption-by-default for details. ([#76310](https://github.com/kubernetes/kubernetes/pull/76310), [@ravisantoshgudimetla](https://github.com/ravisantoshgudimetla))",
-    "author": "ravisantoshgudimetla",
-    "author_url": "https://github.com/ravisantoshgudimetla",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76310",
-    "pr_number": 76310,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "79223": {
-    "commit": "eedfb6bc7a2e3e4d3d69a9fc466dd58795fb0283",
-    "text": "kubeadm no longer defaults or validates the component configs of the kubelet or kube-proxy",
-    "markdown": "kubeadm no longer defaults or validates the component configs of the kubelet or kube-proxy ([#79223](https://github.com/kubernetes/kubernetes/pull/79223), [@rosti](https://github.com/rosti))",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79223",
-    "pr_number": 79223,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.10"
-  },
-  "79993": {
-    "commit": "ded6ee953c68f8333ee6291e0bcb7e58604fac00",
-    "text": "kube-controller-manager\n--node-cidr-mask-size-ipv4 int32     Default: 24. Mask size for IPv4 node-cidr in dual-stack cluster.\n--node-cidr-mask-size-ipv6 int32     Default: 64. Mask size for IPv6 node-cidr in dual-stack cluster.\n\nThese 2 flags can be used only for dual-stack clusters. For non dual-stack clusters, continue to use\n--node-cidr-mask-size flag to configure the mask size.\n\nThe default node cidr mask size for IPv6 was 24 which is now changed to 64.",
-    "markdown": "kube-controller-manager\n  --node-cidr-mask-size-ipv4 int32     Default: 24. Mask size for IPv4 node-cidr in dual-stack cluster.\n  --node-cidr-mask-size-ipv6 int32     Default: 64. Mask size for IPv6 node-cidr in dual-stack cluster.\n  \n  These 2 flags can be used only for dual-stack clusters. For non dual-stack clusters, continue to use\n  --node-cidr-mask-size flag to configure the mask size.\n  \n  The default node cidr mask size for IPv6 was 24 which is now changed to 64. ([#79993](https://github.com/kubernetes/kubernetes/pull/79993), [@aramase](https://github.com/aramase))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Cloud Provider, SIG Network, and SIG Testing",
-    "author": "aramase",
-    "author_url": "https://github.com/aramase",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79993",
-    "pr_number": 79993,
-    "areas": ["test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "cloud-provider", "network", "testing"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "80058": {
-    "commit": "805eecf6922a78a7f1e145f347413aa185b2a942",
-    "text": "Enables VolumeSnapshotDataSource feature gate and promotes volume snapshot APIs to beta.",
-    "markdown": "Enables VolumeSnapshotDataSource feature gate and promotes volume snapshot APIs to beta. ([#80058](https://github.com/kubernetes/kubernetes/pull/80058), [@xing-yang](https://github.com/xing-yang))\n\n  Courtesy of SIG Cluster Lifecycle, SIG Storage, and SIG Testing",
-    "author": "xing-yang",
-    "author_url": "https://github.com/xing-yang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80058",
-    "pr_number": 80058,
-    "areas": ["test"],
-    "kinds": ["bug", "feature"],
-    "sigs": ["cluster-lifecycle", "storage", "testing"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "80284": {
-    "commit": "48ddf3be87789c92e6824c9ce536c76d008f5c19",
-    "text": "Fix a racing issue in client-go UpdateTransportConfig.",
-    "markdown": "Fix a racing issue in client-go UpdateTransportConfig. ([#80284](https://github.com/kubernetes/kubernetes/pull/80284), [@danielqsj](https://github.com/danielqsj))",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80284",
-    "pr_number": 80284,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "80572": {
-    "commit": "a6f51da5001e82d965906c46239589cbcba086f0",
-    "text": "Scale custom resource unconditionally if resourceVersion is not provided",
-    "markdown": "Scale custom resource unconditionally if resourceVersion is not provided ([#80572](https://github.com/kubernetes/kubernetes/pull/80572), [@knight42](https://github.com/knight42))",
-    "author": "knight42",
-    "author_url": "https://github.com/knight42",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80572",
-    "pr_number": 80572,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "80649": {
-    "commit": "bbd48e99b2c19d999dfa25320f055a20deabbc38",
-    "text": "API request throttling (due to a high rate of requests) is now reported in the kubelet (and other component) logs by default.  The messages are of the form\n\nThrottling request took 1.50705208s, request: GET:\u003cURL\u003e\n\nThe presence of large numbers of these messages, particularly with long delay times, may indicate to the administrator the need to tune the cluster accordingly.",
-    "markdown": "API request throttling (due to a high rate of requests) is now reported in the kubelet (and other component) logs by default.  The messages are of the form\n  \n  Throttling request took 1.50705208s, request: GET:\u003cURL\u003e\n  \n  The presence of large numbers of these messages, particularly with long delay times, may indicate to the administrator the need to tune the cluster accordingly. ([#80649](https://github.com/kubernetes/kubernetes/pull/80649), [@RobertKrawitz](https://github.com/RobertKrawitz))\n\n  Courtesy of SIG API Machinery, and SIG Node",
-    "author": "RobertKrawitz",
-    "author_url": "https://github.com/RobertKrawitz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80649",
-    "pr_number": 80649,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "node"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "80854": {
-    "commit": "09d142a7ab093faa8a1efc49e37e45058b4a915a",
-    "text": "Kubenet: added HostPort IPv6 support\nHostPortManager: operates only with one IP family, failing if receives portmapping entries with different IP families\nHostPortSyncer: operates only with one IP family, skipping portmap entries with different IP families",
-    "markdown": "Kubenet: added HostPort IPv6 support\n  HostPortManager: operates only with one IP family, failing if receives portmapping entries with different IP families\n  HostPortSyncer: operates only with one IP family, skipping portmap entries with different IP families ([#80854](https://github.com/kubernetes/kubernetes/pull/80854), [@aojea](https://github.com/aojea))\n\n  Courtesy of SIG Network, and SIG Node",
-    "author": "aojea",
-    "author_url": "https://github.com/aojea",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80854",
-    "pr_number": 80854,
-    "areas": ["kubelet"],
-    "kinds": ["bug", "feature"],
-    "sigs": ["network", "node"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "81319": {
-    "commit": "2bc3804a0e492aa21a368bde054f150edda2a346",
-    "text": "kubeadm: add a upgrade health check that deploys a Job",
-    "markdown": "kubeadm: add a upgrade health check that deploys a Job ([#81319](https://github.com/kubernetes/kubernetes/pull/81319), [@neolit123](https://github.com/neolit123))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81319",
-    "pr_number": 81319,
-    "areas": ["kubeadm", "security"],
-    "kinds": ["cleanup", "feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "82660": {
-    "commit": "646afd52ff93b069401a61dc3b4b9ec37a1d936a",
-    "text": "kubectl drain node --dry-run will list pods that would be evicted or deleted",
-    "markdown": "kubectl drain node --dry-run will list pods that would be evicted or deleted ([#82660](https://github.com/kubernetes/kubernetes/pull/82660), [@sallyom](https://github.com/sallyom))",
-    "author": "sallyom",
-    "author_url": "https://github.com/sallyom",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82660",
-    "pr_number": 82660,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.14.10"
-  },
-  "82690": {
-    "commit": "b32a3d9ca4c94e1105050edcd27456bb3751498c",
-    "text": "The ResourceQuotaScopeSelectors feature has graduated to GA. The `ResourceQuotaScopeSelectors` feature gate is now unconditionally enabled and will be removed in 1.18.",
-    "markdown": "The ResourceQuotaScopeSelectors feature has graduated to GA. The `ResourceQuotaScopeSelectors` feature gate is now unconditionally enabled and will be removed in 1.18. ([#82690](https://github.com/kubernetes/kubernetes/pull/82690), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Scheduling",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-scheduling/20190409-resource-quota-ga.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82690",
-    "pr_number": 82690,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "82966": {
-    "commit": "be4683ef9cc68f50f0f970f036167d84b85d8ed3",
-    "text": "The official kube-proxy image (used by kubeadm, among other things) is now\ncompatible with systems running iptables 1.8 in \"nft\" mode, and will autodetect\nwhich mode it should use.",
-    "markdown": "The official kube-proxy image (used by kubeadm, among other things) is now\n  compatible with systems running iptables 1.8 in \"nft\" mode, and will autodetect\n  which mode it should use. ([#82966](https://github.com/kubernetes/kubernetes/pull/82966), [@danwinship](https://github.com/danwinship))\n\n  Courtesy of SIG Network",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82966",
-    "pr_number": 82966,
-    "kinds": ["feature"],
-    "sigs": ["network"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "83057": {
-    "commit": "d3593c07de4c6a6ef65a54200b3dd0c8d9228f95",
-    "text": "When using Containerd on Windows, the ``TerminationMessagePath`` file will now be mounted in the Windows Pod.",
-    "markdown": "When using Containerd on Windows, the ``TerminationMessagePath`` file will now be mounted in the Windows Pod. ([#83057](https://github.com/kubernetes/kubernetes/pull/83057), [@claudiubelu](https://github.com/claudiubelu))",
-    "author": "claudiubelu",
-    "author_url": "https://github.com/claudiubelu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83057",
-    "pr_number": 83057,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "83362": {
-    "commit": "082883cba32c61b69bdb6709d0132217b4d73dc6",
-    "text": "Bind metrics-server containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes",
-    "markdown": "Bind metrics-server containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes ([#83362](https://github.com/kubernetes/kubernetes/pull/83362), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83362",
-    "pr_number": 83362,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "instrumentation", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "83363": {
-    "commit": "332f452198a8d2fbb2ca20e937a050cf213fac01",
-    "text": "Bind metadata-agent containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes",
-    "markdown": "Bind metadata-agent containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes ([#83363](https://github.com/kubernetes/kubernetes/pull/83363), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83363",
-    "pr_number": 83363,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "instrumentation", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "83394": {
-    "commit": "5e30d32d2cb5cab6f23269a219b59d69a2b969eb",
-    "text": "Existing PVs are converted to use volume topology if migration is enabled.",
-    "markdown": "Existing PVs are converted to use volume topology if migration is enabled. ([#83394](https://github.com/kubernetes/kubernetes/pull/83394), [@bertinatto](https://github.com/bertinatto))\n\n  Courtesy of SIG Apps, SIG Scheduling, and SIG Storage",
-    "author": "bertinatto",
-    "author_url": "https://github.com/bertinatto",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83394",
-    "pr_number": 83394,
-    "kinds": ["feature"],
-    "sigs": ["apps", "scheduling", "storage"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "83568": {
-    "commit": "c580a12c8ea0797c25061a3ff170dcc00e05be79",
-    "text": "Node-specific volume limits has graduated to GA.",
-    "markdown": "Node-specific volume limits has graduated to GA. ([#83568](https://github.com/kubernetes/kubernetes/pull/83568), [@bertinatto](https://github.com/bertinatto))\n\n  Courtesy of SIG Auth, SIG Node, SIG Scheduling, SIG Storage, and SIG Testing",
-    "author": "bertinatto",
-    "author_url": "https://github.com/bertinatto",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83568",
-    "pr_number": 83568,
-    "areas": ["dependency", "kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["auth", "node", "scheduling", "storage", "testing"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "83576": {
-    "commit": "b0c26e5a5af611bf7577a80329889341f21515cf",
-    "text": "update github.com/vishvananda/netlink to v1.0.0",
-    "markdown": "update github.com/vishvananda/netlink to v1.0.0 ([#83576](https://github.com/kubernetes/kubernetes/pull/83576), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83576",
-    "pr_number": 83576,
-    "areas": ["dependency"],
-    "kinds": ["cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.14.10"
-  },
-  "83815": {
-    "commit": "7d1f9b440ef5ce93c43bd07fe4a760c7325dffb6",
-    "text": "Added appProtocol field to EndpointSlice Port",
-    "markdown": "Added appProtocol field to EndpointSlice Port ([#83815](https://github.com/kubernetes/kubernetes/pull/83815), [@howardjohn](https://github.com/howardjohn))",
-    "author": "howardjohn",
-    "author_url": "https://github.com/howardjohn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83815",
-    "pr_number": 83815,
-    "areas": ["dependency"],
-    "kinds": ["api-change"],
-    "sigs": ["apps", "cluster-lifecycle", "instrumentation", "network"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "83836": {
-    "commit": "e45e6e5f2008faba76f03072f6af48ada5e8a50f",
-    "text": "Deprecated metric `rest_client_request_latency_seconds` has been turned off.",
-    "markdown": "Deprecated metric `rest_client_request_latency_seconds` has been turned off. ([#83836](https://github.com/kubernetes/kubernetes/pull/83836), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG API Machinery, SIG Cluster Lifecycle, and SIG Instrumentation",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/da4b7050ccae7a947e4d60f94ab28513e513a458/keps/sig-instrumentation/20181106-kubernetes-metrics-overhaul.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83836",
-    "pr_number": 83836,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "instrumentation"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "83837": {
-    "commit": "beaf3a2f04f2882b9e1de99e4b4f2f06995e3d82",
-    "text": "Following metrics have been turned off:\n- apiserver_request_count\n- apiserver_request_latencies\n- apiserver_request_latencies_summary\n- apiserver_dropped_requests\n- etcd_request_latencies_summary\n- apiserver_storage_transformation_latencies_microseconds\n- apiserver_storage_data_key_generation_latencies_microseconds\n- apiserver_storage_transformation_failures_total",
-    "markdown": "Following metrics have been turned off:\n  - apiserver_request_count\n  - apiserver_request_latencies\n  - apiserver_request_latencies_summary\n  - apiserver_dropped_requests\n  - etcd_request_latencies_summary\n  - apiserver_storage_transformation_latencies_microseconds\n  - apiserver_storage_data_key_generation_latencies_microseconds\n  - apiserver_storage_transformation_failures_total ([#83837](https://github.com/kubernetes/kubernetes/pull/83837), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG API Machinery, and SIG Testing",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/da4b7050ccae7a947e4d60f94ab28513e513a458/keps/sig-instrumentation/20181106-kubernetes-metrics-overhaul.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83837",
-    "pr_number": 83837,
-    "areas": ["apiserver", "e2e-test-framework", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "testing"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "83838": {
-    "commit": "7a5929da02f9fbdb9d63b5335f159462cc553020",
-    "text": "Following metrics have been turned off:\n- scheduler_scheduling_latency_seconds\n- scheduler_e2e_scheduling_latency_microseconds\n- scheduler_scheduling_algorithm_latency_microseconds\n- scheduler_scheduling_algorithm_predicate_evaluation\n- scheduler_scheduling_algorithm_priority_evaluation\n- scheduler_scheduling_algorithm_preemption_evaluation\n- scheduler_scheduling_binding_latency_microseconds",
-    "markdown": "Following metrics have been turned off:\n  - scheduler_scheduling_latency_seconds\n  - scheduler_e2e_scheduling_latency_microseconds\n  - scheduler_scheduling_algorithm_latency_microseconds\n  - scheduler_scheduling_algorithm_predicate_evaluation\n  - scheduler_scheduling_algorithm_priority_evaluation\n  - scheduler_scheduling_algorithm_preemption_evaluation\n  - scheduler_scheduling_binding_latency_microseconds ([#83838](https://github.com/kubernetes/kubernetes/pull/83838), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Scheduling",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/da4b7050ccae7a947e4d60f94ab28513e513a458/keps/sig-instrumentation/20181106-kubernetes-metrics-overhaul.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83838",
-    "pr_number": 83838,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "83839": {
-    "commit": "1ae9713f7f3c8c1bb595a9c04a2f8518c3e5902f",
-    "text": "Deprecated metric `kubeproxy_sync_proxy_rules_latency_microseconds` has been turned off.",
-    "markdown": "Deprecated metric `kubeproxy_sync_proxy_rules_latency_microseconds` has been turned off. ([#83839](https://github.com/kubernetes/kubernetes/pull/83839), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Network",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/da4b7050ccae7a947e4d60f94ab28513e513a458/keps/sig-instrumentation/20181106-kubernetes-metrics-overhaul.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83839",
-    "pr_number": 83839,
-    "kinds": ["feature"],
-    "sigs": ["network"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "83841": {
-    "commit": "ff8cf507dc5476832408ab6a679ffacccfaf2696",
-    "text": "Following metrics have been turned off:\n- kubelet_pod_worker_latency_microseconds\n- kubelet_pod_start_latency_microseconds\n- kubelet_cgroup_manager_latency_microseconds\n- kubelet_pod_worker_start_latency_microseconds\n- kubelet_pleg_relist_latency_microseconds\n- kubelet_pleg_relist_interval_microseconds\n- kubelet_eviction_stats_age_microseconds\n- kubelet_runtime_operations\n- kubelet_runtime_operations_latency_microseconds\n- kubelet_runtime_operations_errors\n- kubelet_device_plugin_registration_count\n- kubelet_device_plugin_alloc_latency_microseconds\n- kubelet_docker_operations\n- kubelet_docker_operations_latency_microseconds\n- kubelet_docker_operations_errors\n- kubelet_docker_operations_timeout\n- network_plugin_operations_latency_microseconds",
-    "markdown": "Following metrics have been turned off:\n  - kubelet_pod_worker_latency_microseconds\n  - kubelet_pod_start_latency_microseconds\n  - kubelet_cgroup_manager_latency_microseconds\n  - kubelet_pod_worker_start_latency_microseconds\n  - kubelet_pleg_relist_latency_microseconds\n  - kubelet_pleg_relist_interval_microseconds\n  - kubelet_eviction_stats_age_microseconds\n  - kubelet_runtime_operations\n  - kubelet_runtime_operations_latency_microseconds\n  - kubelet_runtime_operations_errors\n  - kubelet_device_plugin_registration_count\n  - kubelet_device_plugin_alloc_latency_microseconds\n  - kubelet_docker_operations\n  - kubelet_docker_operations_latency_microseconds\n  - kubelet_docker_operations_errors\n  - kubelet_docker_operations_timeout\n  - network_plugin_operations_latency_microseconds ([#83841](https://github.com/kubernetes/kubernetes/pull/83841), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Network, and SIG Node",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/da4b7050ccae7a947e4d60f94ab28513e513a458/keps/sig-instrumentation/20181106-kubernetes-metrics-overhaul.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83841",
-    "pr_number": 83841,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["network", "node"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "83965": {
-    "commit": "5dc87d2919818bbfd5c68eb1aed766b849a9062c",
-    "text": "Adds a new label to indicate what is managing an EndpointSlice.",
-    "markdown": "Adds a new label to indicate what is managing an EndpointSlice. ([#83965](https://github.com/kubernetes/kubernetes/pull/83965), [@robscott](https://github.com/robscott))\n\n  Courtesy of SIG Apps, and SIG Network",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83965",
-    "pr_number": 83965,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["apps", "network"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "84042": {
-    "commit": "6ff3b68d72089ce4fd0360fca7c1a4f05df834dc",
-    "text": "Removed dependency on kubectl from several storage E2E tests",
-    "markdown": "Removed dependency on kubectl from several storage E2E tests ([#84042](https://github.com/kubernetes/kubernetes/pull/84042), [@okartau](https://github.com/okartau))\n\n  Courtesy of SIG Storage, and SIG Testing",
-    "author": "okartau",
-    "author_url": "https://github.com/okartau",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84042",
-    "pr_number": 84042,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["feature"],
-    "sigs": ["storage", "testing"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "84118": {
-    "commit": "9648d56765bdd77aa504910cdc1795189f11cd51",
-    "text": "kubeadm: add a new \"kubelet-finalize\" phase as part of the \"init\" workflow and an experimental sub-phase to enable automatic kubelet client certificate rotation on primary control-plane nodes.\n\nPrior to 1.17 and for existing nodes created by \"kubeadm init\" where kubelet client certificate rotation is desired, you must modify \"/etc/kubernetes/kubelet.conf\" to point to the PEM symlink for rotation:\n\"client-certificate: /var/lib/kubelet/pki/kubelet-client-current.pem\" and \"client-key: /var/lib/kubelet/pki/kubelet-client-current.pem\", replacing the embedded client certificate and key.",
-    "markdown": "kubeadm: add a new \"kubelet-finalize\" phase as part of the \"init\" workflow and an experimental sub-phase to enable automatic kubelet client certificate rotation on primary control-plane nodes.\n  \n  Prior to 1.17 and for existing nodes created by \"kubeadm init\" where kubelet client certificate rotation is desired, you must modify \"/etc/kubernetes/kubelet.conf\" to point to the PEM symlink for rotation:\n  \"client-certificate: /var/lib/kubelet/pki/kubelet-client-current.pem\" and \"client-key: /var/lib/kubelet/pki/kubelet-client-current.pem\", replacing the embedded client certificate and key. ([#84118](https://github.com/kubernetes/kubernetes/pull/84118), [@neolit123](https://github.com/neolit123))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84118",
-    "pr_number": 84118,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "action_required": true,
-    "release_version": "1.14.10"
-  },
-  "84194": {
-    "commit": "b9030ff666772b76c6bb96c147a52341a9be94b9",
-    "text": "Fix incorrect network policy description suggesting that pods are isolated when a network policy has no rules of a given type",
-    "markdown": "Fix incorrect network policy description suggesting that pods are isolated when a network policy has no rules of a given type ([#84194](https://github.com/kubernetes/kubernetes/pull/84194), [@jackkleeman](https://github.com/jackkleeman))",
-    "author": "jackkleeman",
-    "author_url": "https://github.com/jackkleeman",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84194",
-    "pr_number": 84194,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli", "network"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "84292": {
-    "commit": "21df24cc067f4fd6192cb82992fef0f925b48d19",
-    "text": "New flag `--show-hidden-metrics-for-version` in kube-apiserver can be used to show all hidden metrics that deprecated in the previous minor release.",
-    "markdown": "New flag `--show-hidden-metrics-for-version` in kube-apiserver can be used to show all hidden metrics that deprecated in the previous minor release. ([#84292](https://github.com/kubernetes/kubernetes/pull/84292), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG API Machinery, SIG Cluster Lifecycle, and SIG Instrumentation",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84292",
-    "pr_number": 84292,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "instrumentation"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "84304": {
-    "commit": "f1e912c38abcecfb64e36eb161128b15e69a135b",
-    "text": "`--runtime-config` now supports an `api/beta=false` value which disables all built-in REST API versions matching `v[0-9]+beta[0-9]+`.\n`--feature-gates` now supports an `AllBeta=false` value which disables all beta feature gates.",
-    "markdown": "`--runtime-config` now supports an `api/beta=false` value which disables all built-in REST API versions matching `v[0-9]+beta[0-9]+`.\n  `--feature-gates` now supports an `AllBeta=false` value which disables all beta feature gates. ([#84304](https://github.com/kubernetes/kubernetes/pull/84304), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery, and SIG Cluster Lifecycle",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/pull/1332",
-        "type": "KEP"
-      }
-    ],
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84304",
-    "pr_number": 84304,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "84356": {
-    "commit": "c10d29ba8890d37ddf316a4840860d039a5ab2bc",
-    "text": "Pod process namespace sharing is now Generally Available. The `PodShareProcessNamespace` feature gate is now deprecated and will be removed in Kubernetes 1.19.",
-    "markdown": "Pod process namespace sharing is now Generally Available. The `PodShareProcessNamespace` feature gate is now deprecated and will be removed in Kubernetes 1.19. ([#84356](https://github.com/kubernetes/kubernetes/pull/84356), [@verb](https://github.com/verb))\n\n  Courtesy of SIG Apps, SIG Node, and SIG Testing",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://git.k8s.io/enhancements/keps/sig-node/20190920-pod-pid-namespace.md",
-        "type": "external"
-      },
-      {
-        "description": "[Usage]",
-        "url": "https://kubernetes.io/docs/tasks/configure-pod-container/share-process-namespace",
-        "type": "official"
-      }
-    ],
-    "author": "verb",
-    "author_url": "https://github.com/verb",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84356",
-    "pr_number": 84356,
-    "areas": ["kubelet", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["apps", "node", "testing"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "84390": {
-    "commit": "64f4be5b328a4df8a709b95604743086013a70e4",
-    "text": "EndpointSlices are now beta and enabled by default for better Network Endpoint performance at scale.",
-    "markdown": "EndpointSlices are now beta and enabled by default for better Network Endpoint performance at scale. ([#84390](https://github.com/kubernetes/kubernetes/pull/84390), [@robscott](https://github.com/robscott))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG CLI, SIG Network, and SIG Testing",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84390",
-    "pr_number": 84390,
-    "areas": ["apiserver", "ipvs", "kubectl", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "auth", "cli", "network", "testing"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "84472": {
-    "commit": "42273a4dbf27cd2ed362c1b2e418d2e15fe2c53c",
-    "text": "Adds Windows Server build information as a label on the node.",
-    "markdown": "Adds Windows Server build information as a label on the node. ([#84472](https://github.com/kubernetes/kubernetes/pull/84472), [@gab-satchi](https://github.com/gab-satchi))\n\n  Courtesy of SIG Node, and SIG Windows",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/pull/1302",
-        "type": "KEP"
-      }
-    ],
-    "author": "gab-satchi",
-    "author_url": "https://github.com/gab-satchi",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84472",
-    "pr_number": 84472,
-    "areas": ["kubelet"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["node", "windows"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "84485": {
-    "commit": "0155d18fbc798031fe904bc99abe2b935058beb8",
-    "text": "Mirror pods now include an ownerReference for the node that created them.",
-    "markdown": "Mirror pods now include an ownerReference for the node that created them. ([#84485](https://github.com/kubernetes/kubernetes/pull/84485), [@tallclair](https://github.com/tallclair))\n\n  Courtesy of SIG Node, and SIG Testing",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-auth/20190916-noderestriction-pods.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84485",
-    "pr_number": 84485,
-    "areas": ["kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["node", "testing"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "84522": {
-    "commit": "ce11622bb682027ddd0f195d539d6de7bea2fd01",
-    "text": "Add plugin_execution_duration_seconds metric for scheduler framework plugins.",
-    "markdown": "Add plugin_execution_duration_seconds metric for scheduler framework plugins. ([#84522](https://github.com/kubernetes/kubernetes/pull/84522), [@liu-cong](https://github.com/liu-cong))\n\n  Courtesy of SIG Scheduling",
-    "author": "liu-cong",
-    "author_url": "https://github.com/liu-cong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84522",
-    "pr_number": 84522,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "84524": {
-    "commit": "1ab6360997dda691d6671fc547abbe3776ae8590",
-    "text": "A new flag \"progress-report-url\" has been added to the test context which allows progress information about the test run to be sent to a webhook. In addition, this information is printed to stdout to aid in users watching the logs.",
-    "markdown": "A new flag \"progress-report-url\" has been added to the test context which allows progress information about the test run to be sent to a webhook. In addition, this information is printed to stdout to aid in users watching the logs. ([#84524](https://github.com/kubernetes/kubernetes/pull/84524), [@johnSchnake](https://github.com/johnSchnake))\n\n  Courtesy of SIG Testing",
-    "author": "johnSchnake",
-    "author_url": "https://github.com/johnSchnake",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84524",
-    "pr_number": 84524,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["feature"],
-    "sigs": ["testing"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "84533": {
-    "commit": "897ce3073c056ef2b94708cdd1b56522b6598b54",
-    "text": "BREAKING CHANGE: Remove plugin watching of deprecated directory {kubelet_root_dir}/plugins and CSI V0 support in accordance with deprecation announcement in https://v1-13.docs.kubernetes.io/docs/setup/release/notes/",
-    "markdown": "BREAKING CHANGE: Remove plugin watching of deprecated directory {kubelet_root_dir}/plugins and CSI V0 support in accordance with deprecation announcement in https://v1-13.docs.kubernetes.io/docs/setup/release/notes/ ([#84533](https://github.com/kubernetes/kubernetes/pull/84533), [@davidz627](https://github.com/davidz627))",
-    "author": "davidz627",
-    "author_url": "https://github.com/davidz627",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84533",
-    "pr_number": 84533,
-    "areas": ["dependency", "kubelet", "test"],
-    "kinds": ["cleanup", "documentation"],
-    "sigs": ["node", "storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "84614": {
-    "commit": "205570eb75b52bc0888309c71757314bd099b3d7",
-    "text": "kubelet now exports a \"server_expiration_renew_failure\" and \"client_expiration_renew_failure\" metric counter if the certificate rotations cannot be performed.",
-    "markdown": "kubelet now exports a \"server_expiration_renew_failure\" and \"client_expiration_renew_failure\" metric counter if the certificate rotations cannot be performed. ([#84614](https://github.com/kubernetes/kubernetes/pull/84614), [@rphillips](https://github.com/rphillips))\n\n  Courtesy of SIG API Machinery, SIG Auth, SIG CLI, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Instrumentation, SIG Node, and SIG Release",
-    "author": "rphillips",
-    "author_url": "https://github.com/rphillips",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84614",
-    "pr_number": 84614,
-    "areas": [
-      "apiserver",
-      "cloudprovider",
-      "code-generation",
-      "dependency",
-      "kubectl",
-      "kubelet",
-      "release-eng"
-    ],
-    "kinds": ["feature"],
-    "sigs": [
-      "api-machinery",
-      "auth",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "node",
-      "release"
-    ],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "84648": {
-    "commit": "2f3376e4632a4433c98c49ccaa212216bb2b92bc",
-    "text": "kubeadm: prevent potential hanging of commands such as \"kubeadm reset\" if the apiserver endpoint is not reachable.",
-    "markdown": "kubeadm: prevent potential hanging of commands such as \"kubeadm reset\" if the apiserver endpoint is not reachable. ([#84648](https://github.com/kubernetes/kubernetes/pull/84648), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84648",
-    "pr_number": 84648,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.10"
-  },
-  "84692": {
-    "commit": "c28921f248a8e6c923096154c6e87efcc188b9f0",
-    "text": "Clients can request protobuf and json and correctly negotiate with the server for JSON for CRD objects, allowing all client libraries to request protobuf if it is available.  If an error occurs negotiating a watch with the server, the error is immediately return by the client `Watch()` method instead of being sent as an `Error` event on the watch stream.",
-    "markdown": "Clients can request protobuf and json and correctly negotiate with the server for JSON for CRD objects, allowing all client libraries to request protobuf if it is available.  If an error occurs negotiating a watch with the server, the error is immediately return by the client `Watch()` method instead of being sent as an `Error` event on the watch stream. ([#84692](https://github.com/kubernetes/kubernetes/pull/84692), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84692",
-    "pr_number": 84692,
-    "areas": ["apiserver", "kubectl", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth", "cli", "network", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "84722": {
-    "commit": "8436dde049fb055e2c58c85f3f1dec5c06c36399",
-    "text": "Implement the documented API semantics of list-type and map-type atomic to reject non-atomic sub-types.",
-    "markdown": "Implement the documented API semantics of list-type and map-type atomic to reject non-atomic sub-types. ([#84722](https://github.com/kubernetes/kubernetes/pull/84722), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84722",
-    "pr_number": 84722,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.10"
-  },
-  "84754": {
-    "commit": "8548a253de80c787dbcd2b022faffc75c7c23025",
-    "text": "CSI Migration: Fixes issue where all volumes with the same inline volume inner spec name were staged in the same path. Migrated inline volumes are now staged at a unique path per unique volume.",
-    "markdown": "CSI Migration: Fixes issue where all volumes with the same inline volume inner spec name were staged in the same path. Migrated inline volumes are now staged at a unique path per unique volume. ([#84754](https://github.com/kubernetes/kubernetes/pull/84754), [@davidz627](https://github.com/davidz627))",
-    "author": "davidz627",
-    "author_url": "https://github.com/davidz627",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84754",
-    "pr_number": 84754,
-    "areas": ["dependency"],
-    "kinds": ["bug", "failing-test"],
-    "sigs": ["storage"],
-    "release_version": "1.14.10"
-  },
-  "84768": {
-    "commit": "681d22428b2e037163ea11c13a4c066c2058e515",
-    "text": "- kubelet and aggregated API servers now use v1 TokenReview and SubjectAccessReview endpoints to check authentication/authorization.\n- kube-apiserver can now specify `--authentication-token-webhook-version=v1` or `--authorization-webhook-version=v1` to use `v1` TokenReview and SubjectAccessReview API objects when communicating with authentication and authorization webhooks.",
-    "markdown": "- kubelet and aggregated API servers now use v1 TokenReview and SubjectAccessReview endpoints to check authentication/authorization.\n  - kube-apiserver can now specify `--authentication-token-webhook-version=v1` or `--authorization-webhook-version=v1` to use `v1` TokenReview and SubjectAccessReview API objects when communicating with authentication and authorization webhooks. ([#84768](https://github.com/kubernetes/kubernetes/pull/84768), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84768",
-    "pr_number": 84768,
-    "areas": ["apiserver", "kubelet", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "auth", "node", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "84832": {
-    "commit": "72bcec4e4d516dcac227606541f990af13fb2918",
-    "text": "Bump CSI version to 1.2.0",
-    "markdown": "Bump CSI version to 1.2.0 ([#84832](https://github.com/kubernetes/kubernetes/pull/84832), [@gnufied](https://github.com/gnufied))\n\n  Courtesy of SIG Storage",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84832",
-    "pr_number": 84832,
-    "areas": ["dependency"],
-    "kinds": ["feature"],
-    "sigs": ["storage"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "84882": {
-    "commit": "3eff237357e2df6a64cc14a5dc13c090edafb8be",
-    "text": "Moving WindowsRunAsUserName feature to beta",
-    "markdown": "Moving WindowsRunAsUserName feature to beta ([#84882](https://github.com/kubernetes/kubernetes/pull/84882), [@marosset](https://github.com/marosset))",
-    "author": "marosset",
-    "author_url": "https://github.com/marosset",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84882",
-    "pr_number": 84882,
-    "areas": ["kubelet"],
-    "kinds": ["api-change"],
-    "sigs": ["apps", "node", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "84886": {
-    "commit": "78c56e6b4cf963e5d773908ef9780345d216188b",
-    "text": "Verify kubelet \u0026 kube-proxy can recover after being killed on Windows nodes",
-    "markdown": "Verify kubelet \u0026 kube-proxy can recover after being killed on Windows nodes ([#84886](https://github.com/kubernetes/kubernetes/pull/84886), [@YangLu1031](https://github.com/YangLu1031))",
-    "author": "YangLu1031",
-    "author_url": "https://github.com/YangLu1031",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84886",
-    "pr_number": 84886,
-    "areas": ["test"],
-    "kinds": ["failing-test"],
-    "sigs": ["apps", "testing", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "84907": {
-    "commit": "b3dde20411901a9a6114381811623083e73ed440",
-    "text": "Following metrics from kubelet are now marked as with the ALPHA stability level:\nkubelet_container_log_filesystem_used_bytes\nkubelet_volume_stats_capacity_bytes\nkubelet_volume_stats_available_bytes\nkubelet_volume_stats_used_bytes\nkubelet_volume_stats_inodes\nkubelet_volume_stats_inodes_free\nkubelet_volume_stats_inodes_used\nplugin_manager_total_plugins\nvolume_manager_total_volumes",
-    "markdown": "Following metrics from kubelet are now marked as with the ALPHA stability level:\n  kubelet_container_log_filesystem_used_bytes\n  kubelet_volume_stats_capacity_bytes\n  kubelet_volume_stats_available_bytes\n  kubelet_volume_stats_used_bytes\n  kubelet_volume_stats_inodes\n  kubelet_volume_stats_inodes_free\n  kubelet_volume_stats_inodes_used\n  plugin_manager_total_plugins\n  volume_manager_total_volumes ([#84907](https://github.com/kubernetes/kubernetes/pull/84907), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Node",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84907",
-    "pr_number": 84907,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "84957": {
-    "commit": "35de7a2537f73de74fa22288c09fe927d9cb21d6",
-    "text": "matches := stackCreator.FindStringSubmatch(string(stack))\nmatches type is []string\n\nso nil len(matches) ==0 \n\nchange:\n if matches == nil || len(matches) != 4 ==\u003eif len(matches) != 4",
-    "markdown": "matches := stackCreator.FindStringSubmatch(string(stack))\n  matches type is []string\n  \n  so nil len(matches) ==0 \n  \n  change:\n   if matches == nil || len(matches) != 4 ==\u003eif len(matches) != 4 ([#84957](https://github.com/kubernetes/kubernetes/pull/84957), [@ZP-AlwaysWin](https://github.com/ZP-AlwaysWin))",
-    "author": "ZP-AlwaysWin",
-    "author_url": "https://github.com/ZP-AlwaysWin",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84957",
-    "pr_number": 84957,
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "84971": {
-    "commit": "c5609071d805036b251f22ce122bbf13ca94bff7",
-    "text": "Splitting IP address type into IPv4 and IPv6 for EndpointSlices",
-    "markdown": "Splitting IP address type into IPv4 and IPv6 for EndpointSlices ([#84971](https://github.com/kubernetes/kubernetes/pull/84971), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84971",
-    "pr_number": 84971,
-    "areas": ["dependency", "ipvs", "kubectl", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["apps", "cli", "network", "scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "84987": {
-    "commit": "ec86baf00ba6478362031f32787e69765da4bfeb",
-    "text": "Following metrics from kubelet are now marked as with the ALPHA stability level:\nnode_cpu_usage_seconds_total\nnode_memory_working_set_bytes\ncontainer_cpu_usage_seconds_total\ncontainer_memory_working_set_bytes\nscrape_error",
-    "markdown": "Following metrics from kubelet are now marked as with the ALPHA stability level:\n  node_cpu_usage_seconds_total\n  node_memory_working_set_bytes\n  container_cpu_usage_seconds_total\n  container_memory_working_set_bytes\n  scrape_error ([#84987](https://github.com/kubernetes/kubernetes/pull/84987), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG API Machinery, SIG Cluster Lifecycle, SIG Instrumentation, and SIG Node",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84987",
-    "pr_number": 84987,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "instrumentation", "node"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "84998": {
-    "commit": "43ccfced1dc1db6fcf7209e710021518b3d42ae2",
-    "text": "Fix a bug that a node Lease object may have been created without OwnerReference.",
-    "markdown": "Fix a bug that a node Lease object may have been created without OwnerReference. ([#84998](https://github.com/kubernetes/kubernetes/pull/84998), [@wojtek-t](https://github.com/wojtek-t))",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84998",
-    "pr_number": 84998,
-    "areas": ["kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["node", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85004": {
-    "commit": "02af1dd62c4842e20e2ee7337edf032327b1c8ed",
-    "text": "apiservers based on k8s.io/apiserver with delegated authn based on cluster authentication will automatically update to new authentication information when the authoritative configmap is updated.",
-    "markdown": "apiservers based on k8s.io/apiserver with delegated authn based on cluster authentication will automatically update to new authentication information when the authoritative configmap is updated. ([#85004](https://github.com/kubernetes/kubernetes/pull/85004), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85004",
-    "pr_number": 85004,
-    "areas": ["apiserver", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85023": {
-    "commit": "072cf5bd58b3699701d8f5d85241442c872c7c8b",
-    "text": "Finalizer Protection for Service LoadBalancers is now in GA (enabled by default). This feature ensures the Service resource is not fully deleted until the correlating load balancer resources are deleted.",
-    "markdown": "Finalizer Protection for Service LoadBalancers is now in GA (enabled by default). This feature ensures the Service resource is not fully deleted until the correlating load balancer resources are deleted. ([#85023](https://github.com/kubernetes/kubernetes/pull/85023), [@MrHohn](https://github.com/MrHohn))\n\n  Courtesy of SIG Apps, and SIG Network",
-    "author": "MrHohn",
-    "author_url": "https://github.com/MrHohn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85023",
-    "pr_number": 85023,
-    "kinds": ["feature"],
-    "sigs": ["apps", "network"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "85024": {
-    "commit": "9920b733ba56f908d48dc7d0a30bc90d28d015a4",
-    "text": "kubeadm: fix skipped etcd upgrade on secondary control-plane nodes when the command \"kubeadm upgrade node\" is used.",
-    "markdown": "kubeadm: fix skipped etcd upgrade on secondary control-plane nodes when the command \"kubeadm upgrade node\" is used. ([#85024](https://github.com/kubernetes/kubernetes/pull/85024), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85024",
-    "pr_number": 85024,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.10"
-  },
-  "85043": {
-    "commit": "939e1e64881c0fc920b75c7489816fbf72806358",
-    "text": "kubeadm: enable the usage of the secure kube-scheduler and kube-controller-manager ports for health checks. For kube-scheduler was 10251, becomes 10259. For kube-controller-manager was 10252, becomes 10257.",
-    "markdown": "kubeadm: enable the usage of the secure kube-scheduler and kube-controller-manager ports for health checks. For kube-scheduler was 10251, becomes 10259. For kube-controller-manager was 10252, becomes 10257. ([#85043](https://github.com/kubernetes/kubernetes/pull/85043), [@neolit123](https://github.com/neolit123))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85043",
-    "pr_number": 85043,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "85044": {
-    "commit": "fc0bf06983e32424cc90bff37db561a7dde95943",
-    "text": "kubeadm: remove the deprecated \"--cri-socket\" flag for \"kubeadm upgrade apply\". The flag has been deprecated since v1.14.",
-    "markdown": "kubeadm: remove the deprecated \"--cri-socket\" flag for \"kubeadm upgrade apply\". The flag has been deprecated since v1.14. ([#85044](https://github.com/kubernetes/kubernetes/pull/85044), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85044",
-    "pr_number": 85044,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.10"
-  },
-  "85048": {
-    "commit": "be1658e08c6762d89c7f1fdc0655bb0a45dad1fb",
-    "text": "add RequiresExactMatch for label.Selector",
-    "markdown": "add RequiresExactMatch for label.Selector ([#85048](https://github.com/kubernetes/kubernetes/pull/85048), [@shaloulcy](https://github.com/shaloulcy))\n\n  Courtesy of SIG API Machinery",
-    "author": "shaloulcy",
-    "author_url": "https://github.com/shaloulcy",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85048",
-    "pr_number": 85048,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "85091": {
-    "commit": "4f6aa5a921f9b10ea7dce399f5b3c360953528bb",
-    "text": "Fixed bug when using kubeadm alpha certs commands with clusters using external etcd",
-    "markdown": "Fixed bug when using kubeadm alpha certs commands with clusters using external etcd ([#85091](https://github.com/kubernetes/kubernetes/pull/85091), [@fabriziopandini](https://github.com/fabriziopandini))",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85091",
-    "pr_number": 85091,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.10"
-  },
-  "85092": {
-    "commit": "e008523f76497a31a0bf62602357f5117c5d5f54",
-    "text": "kubeadm alpha certs command now skip missing files",
-    "markdown": "kubeadm alpha certs command now skip missing files ([#85092](https://github.com/kubernetes/kubernetes/pull/85092), [@fabriziopandini](https://github.com/fabriziopandini))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85092",
-    "pr_number": 85092,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "85094": {
-    "commit": "0968636760dae228581b7b515cf605fccd7e2969",
-    "text": "kubeadm deprecates the use of the hyperkube image",
-    "markdown": "kubeadm deprecates the use of the hyperkube image ([#85094](https://github.com/kubernetes/kubernetes/pull/85094), [@rosti](https://github.com/rosti))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85094",
-    "pr_number": 85094,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "action_required": true,
-    "release_version": "1.14.10"
-  },
-  "85098": {
-    "commit": "b49afbfa64b6be72779b6deb424d7f0cddd5ccf1",
-    "text": "kube-apiserver: The `AdmissionConfiguration` type accepted by `--admission-control-config-file` has been promoted to `apiserver.config.k8s.io/v1` with no schema changes.",
-    "markdown": "kube-apiserver: The `AdmissionConfiguration` type accepted by `--admission-control-config-file` has been promoted to `apiserver.config.k8s.io/v1` with no schema changes. ([#85098](https://github.com/kubernetes/kubernetes/pull/85098), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85098",
-    "pr_number": 85098,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.10"
-  },
-  "85099": {
-    "commit": "195664db0eb633903f0098bd07ded2898aa40e9a",
-    "text": "kube-apiserver: The `ResourceQuota` admission plugin configuration referenced from `--admission-control-config-file` admission config has been promoted to `apiVersion: apiserver.config.k8s.io/v1`, `kind: ResourceQuotaConfiguration` with no schema changes.",
-    "markdown": "kube-apiserver: The `ResourceQuota` admission plugin configuration referenced from `--admission-control-config-file` admission config has been promoted to `apiVersion: apiserver.config.k8s.io/v1`, `kind: ResourceQuotaConfiguration` with no schema changes. ([#85099](https://github.com/kubernetes/kubernetes/pull/85099), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85099",
-    "pr_number": 85099,
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.10"
-  },
-  "85109": {
-    "commit": "d0f021524e908c8b646563eb1864a0d3782db963",
-    "text": "Kubeadm now includes CoreDNS version 1.6.5\n - `kubernetes` plugin adds metrics to measure kubernetes control plane latency.\n -  the `health` plugin now includes the `lameduck` option by default, which waits for a duration before shutting down.",
-    "markdown": "Kubeadm now includes CoreDNS version 1.6.5\n   - `kubernetes` plugin adds metrics to measure kubernetes control plane latency.\n   -  the `health` plugin now includes the `lameduck` option by default, which waits for a duration before shutting down. ([#85109](https://github.com/kubernetes/kubernetes/pull/85109), [@rajansandeep](https://github.com/rajansandeep))",
-    "author": "rajansandeep",
-    "author_url": "https://github.com/rajansandeep",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85109",
-    "pr_number": 85109,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.10"
-  },
-  "85115": {
-    "commit": "5dd641e45cc29957db2f75d64a41c782b08a953d",
-    "text": "azure: update disk lock logic per vm during attach/detach to allow concurrent updates for different nodes.",
-    "markdown": "azure: update disk lock logic per vm during attach/detach to allow concurrent updates for different nodes. ([#85115](https://github.com/kubernetes/kubernetes/pull/85115), [@aramase](https://github.com/aramase))",
-    "author": "aramase",
-    "author_url": "https://github.com/aramase",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85115",
-    "pr_number": 85115,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.10"
-  },
-  "85138": {
-    "commit": "f501d8e59a599eaad45a50ee1da075cedc9d0ab9",
-    "text": "kube-apiserver: Authentication configuration for mutating and validating admission webhooks referenced from an `--admission-control-config-file` can now be specified with `apiVersion: apiserver.config.k8s.io/v1, kind: WebhookAdmissionConfiguration`.",
-    "markdown": "kube-apiserver: Authentication configuration for mutating and validating admission webhooks referenced from an `--admission-control-config-file` can now be specified with `apiVersion: apiserver.config.k8s.io/v1, kind: WebhookAdmissionConfiguration`. ([#85138](https://github.com/kubernetes/kubernetes/pull/85138), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85138",
-    "pr_number": 85138,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.10"
-  },
-  "85156": {
-    "commit": "f5df681b809f774056d934294f87858909b8a781",
-    "text": "Fixes a bug in kubeadm that caused init and join to hang indefinitely in specific conditions.",
-    "markdown": "Fixes a bug in kubeadm that caused init and join to hang indefinitely in specific conditions. ([#85156](https://github.com/kubernetes/kubernetes/pull/85156), [@chuckha](https://github.com/chuckha))",
-    "author": "chuckha",
-    "author_url": "https://github.com/chuckha",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85156",
-    "pr_number": 85156,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.10"
-  },
-  "85158": {
-    "commit": "1057f3a82ac4460ca40753614962c39f238913b6",
-    "text": "fix vmss dirty cache issue in disk attach/detach on vmss node",
-    "markdown": "fix vmss dirty cache issue in disk attach/detach on vmss node ([#85158](https://github.com/kubernetes/kubernetes/pull/85158), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85158",
-    "pr_number": 85158,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.10"
-  },
-  "85174": {
-    "commit": "547fdcc1640f614c21d8cc1040c7f9579c68c0c0",
-    "text": "Add table convertor to componentstatus.",
-    "markdown": "Add table convertor to componentstatus. ([#85174](https://github.com/kubernetes/kubernetes/pull/85174), [@zhouya0](https://github.com/zhouya0))",
-    "author": "zhouya0",
-    "author_url": "https://github.com/zhouya0",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85174",
-    "pr_number": 85174,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85201": {
-    "commit": "d1e8702d36947ccc23ad820eaae7f4afbaf0b058",
-    "text": "kubeadm: added retry to all the calls to the etcd API so kubeadm will be more resilient to network glitches",
-    "markdown": "kubeadm: added retry to all the calls to the etcd API so kubeadm will be more resilient to network glitches ([#85201](https://github.com/kubernetes/kubernetes/pull/85201), [@fabriziopandini](https://github.com/fabriziopandini))",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85201",
-    "pr_number": 85201,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.10"
-  },
-  "85223": {
-    "commit": "976712556e4bd22d5312a0af36b18127c709d54a",
-    "text": "CRDs can have fields named `type` with value `array` and nested array with `items` fields without validation to fall over this.",
-    "markdown": "CRDs can have fields named `type` with value `array` and nested array with `items` fields without validation to fall over this. ([#85223](https://github.com/kubernetes/kubernetes/pull/85223), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85223",
-    "pr_number": 85223,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl"],
-    "kinds": ["bug"],
-    "sigs": [
-      "api-machinery",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "node"
-    ],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85228": {
-    "commit": "02c61b6af1250bfabf99628870cfa689bd790c62",
-    "text": "Wait for kubelet \u0026 kube-proxy to be ready on Windows node within 10s",
-    "markdown": "Wait for kubelet \u0026 kube-proxy to be ready on Windows node within 10s ([#85228](https://github.com/kubernetes/kubernetes/pull/85228), [@YangLu1031](https://github.com/YangLu1031))",
-    "author": "YangLu1031",
-    "author_url": "https://github.com/YangLu1031",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85228",
-    "pr_number": 85228,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.10"
-  },
-  "85231": {
-    "commit": "9c26a752a9cfa45423ba2d8148b16b22de31e65b",
-    "text": "Feature gates CSIMigration to Beta (on by default) and CSIMigrationGCE to Beta (off by default since it requires installation of the GCE PD CSI Driver)\nThe in-tree GCE PD plugin \"kubernetes.io/gce-pd\" is now deprecated and will be removed in 1.21. Users should enable CSIMigration + CSIMigrationGCE features and install the GCE PD CSI Driver (https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) to avoid disruption to existing Pod and PVC objects at that time.\nUsers should start using the GCE PD CSI CSI Driver directly for any new volumes.",
-    "markdown": "Feature gates CSIMigration to Beta (on by default) and CSIMigrationGCE to Beta (off by default since it requires installation of the GCE PD CSI Driver)\n  The in-tree GCE PD plugin \"kubernetes.io/gce-pd\" is now deprecated and will be removed in 1.21. Users should enable CSIMigration + CSIMigrationGCE features and install the GCE PD CSI Driver (https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) to avoid disruption to existing Pod and PVC objects at that time.\n  Users should start using the GCE PD CSI CSI Driver directly for any new volumes. ([#85231](https://github.com/kubernetes/kubernetes/pull/85231), [@davidz627](https://github.com/davidz627))\n\n  Courtesy of SIG Apps, SIG Auth, SIG Node, SIG Storage, and SIG Testing",
-    "author": "davidz627",
-    "author_url": "https://github.com/davidz627",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85231",
-    "pr_number": 85231,
-    "areas": ["kubelet", "test"],
-    "kinds": ["documentation", "feature"],
-    "sigs": ["apps", "auth", "node", "storage", "testing"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "85237": {
-    "commit": "81af5baff8e958dc8817913ec3c4172028f30122",
-    "text": "Promote CSIMigrationAWS to Beta (off by default since it requires installation of the AWS EBS CSI Driver)\nThe in-tree AWS EBS plugin \"kubernetes.io/aws-ebs\" is now deprecated and will be removed in 1.21. Users should enable CSIMigration + CSIMigrationAWS features and install the AWS EBS CSI Driver (https://github.com/kubernetes-sigs/aws-ebs-csi-driver) to avoid disruption to existing Pod and PVC objects at that time.\nUsers should start using the AWS EBS CSI CSI Driver directly for any new volumes.",
-    "markdown": "Promote CSIMigrationAWS to Beta (off by default since it requires installation of the AWS EBS CSI Driver)\n  The in-tree AWS EBS plugin \"kubernetes.io/aws-ebs\" is now deprecated and will be removed in 1.21. Users should enable CSIMigration + CSIMigrationAWS features and install the AWS EBS CSI Driver (https://github.com/kubernetes-sigs/aws-ebs-csi-driver) to avoid disruption to existing Pod and PVC objects at that time.\n  Users should start using the AWS EBS CSI CSI Driver directly for any new volumes. ([#85237](https://github.com/kubernetes/kubernetes/pull/85237), [@leakingtapan](https://github.com/leakingtapan))\n\n  Courtesy of SIG Storage",
-    "author": "leakingtapan",
-    "author_url": "https://github.com/leakingtapan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85237",
-    "pr_number": 85237,
-    "areas": ["provider/aws"],
-    "kinds": ["feature"],
-    "sigs": ["storage"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "85246": {
-    "commit": "27067540ffc97495e7a63f402b4a2fda01f40085",
-    "text": "kube-proxy now supports DualStack feature with EndpointSlices and IPVS.",
-    "markdown": "kube-proxy now supports DualStack feature with EndpointSlices and IPVS. ([#85246](https://github.com/kubernetes/kubernetes/pull/85246), [@robscott](https://github.com/robscott))\n\n  Courtesy of SIG Network",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85246",
-    "pr_number": 85246,
-    "areas": ["ipvs"],
-    "kinds": ["feature"],
-    "sigs": ["network"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "85260": {
-    "commit": "9d0a32caf8bf9f2fdc31ee586938bdbaba64fab4",
-    "text": "Renamed FeatureGate RequestManagement to APIPriorityAndFairness.  This feature gate is an alpha and has not yet been associated with any actual functionality.\nchange references to feature gate RequestManagement into references to APIPriorityAndFairness",
-    "markdown": "Renamed FeatureGate RequestManagement to APIPriorityAndFairness.  This feature gate is an alpha and has not yet been associated with any actual functionality.\n  change references to feature gate RequestManagement into references to APIPriorityAndFairness ([#85260](https://github.com/kubernetes/kubernetes/pull/85260), [@MikeSpreitzer](https://github.com/MikeSpreitzer))\n\n  Courtesy of SIG API Machinery",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190228-priority-and-fairness.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "MikeSpreitzer",
-    "author_url": "https://github.com/MikeSpreitzer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85260",
-    "pr_number": 85260,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "action_required": true,
-    "release_version": "1.14.10"
-  },
-  "85265": {
-    "commit": "f01a4d06085b7b44844c472d6dedbd0538080c78",
-    "text": "kubeadm: reset raises warnings if it cannot delete folders",
-    "markdown": "kubeadm: reset raises warnings if it cannot delete folders ([#85265](https://github.com/kubernetes/kubernetes/pull/85265), [@SataQiu](https://github.com/SataQiu))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85265",
-    "pr_number": 85265,
-    "areas": ["kubeadm"],
-    "kinds": ["bug", "feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "85285": {
-    "commit": "141329fd2157a0791b93fae0160ce24d99808607",
-    "text": "kubectl: --resource-version now works properly in label/annotate/set selector commands when racing with other clients to update the target object",
-    "markdown": "kubectl: --resource-version now works properly in label/annotate/set selector commands when racing with other clients to update the target object ([#85285](https://github.com/kubernetes/kubernetes/pull/85285), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85285",
-    "pr_number": 85285,
-    "areas": ["kubectl", "test"],
-    "kinds": ["bug"],
-    "sigs": ["cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85293": {
-    "commit": "2b6857db4d4226cc07180308fe8a8507abbc7700",
-    "text": "Includes FSType when describing CSI persistent volumes.",
-    "markdown": "Includes FSType when describing CSI persistent volumes. ([#85293](https://github.com/kubernetes/kubernetes/pull/85293), [@huffmanca](https://github.com/huffmanca))",
-    "author": "huffmanca",
-    "author_url": "https://github.com/huffmanca",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85293",
-    "pr_number": 85293,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85346": {
-    "commit": "f16bfb069a22241a5501f6fe530f5d4e2a82cf0e",
-    "text": "Remove redundant API validation when using Service Topology with externalTrafficPolicy=Local",
-    "markdown": "Remove redundant API validation when using Service Topology with externalTrafficPolicy=Local ([#85346](https://github.com/kubernetes/kubernetes/pull/85346), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85346",
-    "pr_number": 85346,
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.14.10"
-  },
-  "85351": {
-    "commit": "01e1dcf429cf75a6fed447fc4ff30b8a17df9c83",
-    "text": "kubeadm: Fix a bug where kubeadm cannot parse kubelet's version if the latter dumps logs on the standard error.",
-    "markdown": "kubeadm: Fix a bug where kubeadm cannot parse kubelet's version if the latter dumps logs on the standard error. ([#85351](https://github.com/kubernetes/kubernetes/pull/85351), [@rosti](https://github.com/rosti))",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85351",
-    "pr_number": 85351,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.10"
-  },
-  "85359": {
-    "commit": "f2469a7a2c75a78b8c0b0514652511d105d2b2e7",
-    "text": "When upgrading to 1.17 with a cluster with EndpointSlices enabled, the `endpointslice.kubernetes.io/managed-by` label needs to be set on each EndpointSlice.",
-    "markdown": "When upgrading to 1.17 with a cluster with EndpointSlices enabled, the `endpointslice.kubernetes.io/managed-by` label needs to be set on each EndpointSlice. ([#85359](https://github.com/kubernetes/kubernetes/pull/85359), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85359",
-    "pr_number": 85359,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "network", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85361": {
-    "commit": "9859e73433095b8f05695c6ec450215d173cf27a",
-    "text": "kube-controller-manager: Fixes bug setting headless service labels on endpoints",
-    "markdown": "kube-controller-manager: Fixes bug setting headless service labels on endpoints ([#85361](https://github.com/kubernetes/kubernetes/pull/85361), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85361",
-    "pr_number": 85361,
-    "kinds": ["bug"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85363": {
-    "commit": "0810bc33869b9b2e5bbabef4fa41e99d53526cdb",
-    "text": "action required\n1. Currently, if users were to explicitly specify CacheSize of 0 for KMS provider, they would end-up with a provider that caches up to 1000 keys. This PR changes this behavior.\nPost this PR, when users supply 0 for CacheSize this will result in a validation error.\n\n2. CacheSize type was changed from int32 to *int32. This allows defaulting logic to differentiate between cases where users explicitly supplied 0 vs. not supplied any value.\n3. KMS Provider's endpoint (path to Unix socket) is now validated when the EncryptionConfiguration files is loaded. This used to be handled by the GRPCService.",
-    "markdown": "action required\n  1. Currently, if users were to explicitly specify CacheSize of 0 for KMS provider, they would end-up with a provider that caches up to 1000 keys. This PR changes this behavior.\n  Post this PR, when users supply 0 for CacheSize this will result in a validation error.\n  \n  2. CacheSize type was changed from int32 to *int32. This allows defaulting logic to differentiate between cases where users explicitly supplied 0 vs. not supplied any value.\n  3. KMS Provider's endpoint (path to Unix socket) is now validated when the EncryptionConfiguration files is loaded. This used to be handled by the GRPCService. ([#85363](https://github.com/kubernetes/kubernetes/pull/85363), [@immutableT](https://github.com/immutableT))\n\n  Courtesy of SIG API Machinery, SIG Auth, and SIG Instrumentation",
-    "author": "immutableT",
-    "author_url": "https://github.com/immutableT",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85363",
-    "pr_number": 85363,
-    "areas": ["apiserver"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "auth", "instrumentation"],
-    "action_required": true,
-    "release_version": "1.14.10"
-  },
-  "85365": {
-    "commit": "5848ee494593dea2556a81c0f7a69ac3f7754d1a",
-    "text": "EndpointSlices are not enabled by default. Use the EndpointSlice feature gate to enable this feature.",
-    "markdown": "EndpointSlices are not enabled by default. Use the EndpointSlice feature gate to enable this feature. ([#85365](https://github.com/kubernetes/kubernetes/pull/85365), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85365",
-    "pr_number": 85365,
-    "kinds": ["bug"],
-    "sigs": ["auth", "network"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85368": {
-    "commit": "9905a33fd738a52d3f7ae73cf823d82a05720e33",
-    "text": "Fix bug where EndpointSlice controller would attempt to modify shared objects.",
-    "markdown": "Fix bug where EndpointSlice controller would attempt to modify shared objects. ([#85368](https://github.com/kubernetes/kubernetes/pull/85368), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85368",
-    "pr_number": 85368,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "apps", "network"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85381": {
-    "commit": "ac2a1b7837d0d38cbe57a9d51387737bfb7a6a32",
-    "text": "OpenAPI v3 format in CustomResourceDefinition schemas are now documented.",
-    "markdown": "OpenAPI v3 format in CustomResourceDefinition schemas are now documented. ([#85381](https://github.com/kubernetes/kubernetes/pull/85381), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85381",
-    "pr_number": 85381,
-    "areas": ["test"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85408": {
-    "commit": "bf8c276019331f589dbdd2007e552a98d2e50eed",
-    "text": "Use context to check client closed instead of http.CloseNotifier in processing watch request which will reduce 1 goroutine for each request if proto is HTTP/2.x .",
-    "markdown": "Use context to check client closed instead of http.CloseNotifier in processing watch request which will reduce 1 goroutine for each request if proto is HTTP/2.x . ([#85408](https://github.com/kubernetes/kubernetes/pull/85408), [@answer1991](https://github.com/answer1991))",
-    "author": "answer1991",
-    "author_url": "https://github.com/answer1991",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85408",
-    "pr_number": 85408,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.10"
-  },
-  "85410": {
-    "commit": "34f349257f4fc14c1adc8d9a62109216a6955035",
-    "text": "Fix API Server potential memory leak issue in processing watch request.",
-    "markdown": "Fix API Server potential memory leak issue in processing watch request. ([#85410](https://github.com/kubernetes/kubernetes/pull/85410), [@answer1991](https://github.com/answer1991))",
-    "author": "answer1991",
-    "author_url": "https://github.com/answer1991",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85410",
-    "pr_number": 85410,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.10"
-  },
-  "85412": {
-    "commit": "c65c56348760b5a3c10d9ec214f99ddb3add1488",
-    "text": "kubeadm: don't write the kubelet environment file on \"upgrade apply\"",
-    "markdown": "kubeadm: don't write the kubelet environment file on \"upgrade apply\" ([#85412](https://github.com/kubernetes/kubernetes/pull/85412), [@boluisa](https://github.com/boluisa))",
-    "author": "boluisa",
-    "author_url": "https://github.com/boluisa",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85412",
-    "pr_number": 85412,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.10"
-  },
-  "85441": {
-    "commit": "384e45febdb302c145ab0e1fd3bcf0eda882e2e8",
-    "text": "Resolves error from v1.17.0-beta.2 with --authorizer-mode webhook complaining about an invalid version",
-    "markdown": "Resolves error from v1.17.0-beta.2 with --authorizer-mode webhook complaining about an invalid version ([#85441](https://github.com/kubernetes/kubernetes/pull/85441), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85441",
-    "pr_number": 85441,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85444": {
-    "commit": "5d3125e51258646fff19089f9eb2f2f80c784385",
-    "text": "kube-apiserver: Fixes a bug that hidden metrics can not be enabled by the command-line option `--show-hidden-metrics-for-version`.",
-    "markdown": "kube-apiserver: Fixes a bug that hidden metrics can not be enabled by the command-line option `--show-hidden-metrics-for-version`. ([#85444](https://github.com/kubernetes/kubernetes/pull/85444), [@RainbowMango](https://github.com/RainbowMango))",
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85444",
-    "pr_number": 85444,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85446": {
-    "commit": "c7a65ca0c352d58d94327ed80f35757bef3d6792",
-    "text": "kubelet: the metric process_start_time_seconds be marked as with the ALPHA stability level.",
-    "markdown": "kubelet: the metric process_start_time_seconds be marked as with the ALPHA stability level. ([#85446](https://github.com/kubernetes/kubernetes/pull/85446), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG API Machinery, SIG Cluster Lifecycle, SIG Instrumentation, and SIG Node",
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85446",
-    "pr_number": 85446,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "instrumentation", "node"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "85463": {
-    "commit": "4000732bec17d16ad183e38774883f8a6ae93fd4",
-    "text": "SafeSysctlWhitelist: add net.ipv4.ping_group_range",
-    "markdown": "SafeSysctlWhitelist: add net.ipv4.ping_group_range ([#85463](https://github.com/kubernetes/kubernetes/pull/85463), [@AkihiroSuda](https://github.com/AkihiroSuda))\n\n  Courtesy of SIG Auth",
-    "author": "AkihiroSuda",
-    "author_url": "https://github.com/AkihiroSuda",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85463",
-    "pr_number": 85463,
-    "kinds": ["feature"],
-    "sigs": ["auth"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "85466": {
-    "commit": "d9ed6110e88e73da597cc03a38ee1357f9b67ed3",
-    "text": "Windows nodes on GCE can use TPM-based authentication to the master.",
-    "markdown": "Windows nodes on GCE can use TPM-based authentication to the master. ([#85466](https://github.com/kubernetes/kubernetes/pull/85466), [@pjh](https://github.com/pjh))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "pjh",
-    "author_url": "https://github.com/pjh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85466",
-    "pr_number": 85466,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "85475": {
-    "commit": "97e34ee223c1a7e2c83399d789c8139d29be88aa",
-    "text": "fix azure file AuthorizationFailure",
-    "markdown": "fix azure file AuthorizationFailure ([#85475](https://github.com/kubernetes/kubernetes/pull/85475), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85475",
-    "pr_number": 85475,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85490": {
-    "commit": "7c87b5fb55ca096c007c8739d4657a5a4e29fb09",
-    "text": "kubectl set resources will no longer return an error if passed an empty change for a resource. \nkubectl set subject will no longer return an error if passed an empty change for a resource.",
-    "markdown": "kubectl set resources will no longer return an error if passed an empty change for a resource. \n  kubectl set subject will no longer return an error if passed an empty change for a resource. ([#85490](https://github.com/kubernetes/kubernetes/pull/85490), [@sallyom](https://github.com/sallyom))",
-    "author": "sallyom",
-    "author_url": "https://github.com/sallyom",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85490",
-    "pr_number": 85490,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.10"
-  },
-  "85494": {
-    "commit": "3c5dad61f72cc0fcee20bc5c27db1bde31a691b9",
-    "text": "kubeadm: fix stray \"node-cidr-mask-size\" flag in the kube-controller-manager manifest when IPv6DualStack is enabled",
-    "markdown": "kubeadm: fix stray \"node-cidr-mask-size\" flag in the kube-controller-manager manifest when IPv6DualStack is enabled ([#85494](https://github.com/kubernetes/kubernetes/pull/85494), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85494",
-    "pr_number": 85494,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.10"
-  },
-  "85516": {
-    "commit": "1682a4a0b91edc64b42728a2b92aefe48d69ccdb",
-    "text": "kube-apiserver: fixed a conflict error encountered attempting to delete a pod with gracePeriodSeconds=0 and a resourceVersion precondition",
-    "markdown": "kube-apiserver: fixed a conflict error encountered attempting to delete a pod with gracePeriodSeconds=0 and a resourceVersion precondition ([#85516](https://github.com/kubernetes/kubernetes/pull/85516), [@michaelgugino](https://github.com/michaelgugino))",
-    "author": "michaelgugino",
-    "author_url": "https://github.com/michaelgugino",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85516",
-    "pr_number": 85516,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.10"
-  },
-  "85524": {
-    "commit": "3028dc1f7ac6dfd87463e6d21466966984990bf5",
-    "text": "kubeadm: fix a panic in case the KubeProxyConfiguration feature gates were not initialized.",
-    "markdown": "kubeadm: fix a panic in case the KubeProxyConfiguration feature gates were not initialized. ([#85524](https://github.com/kubernetes/kubernetes/pull/85524), [@Arvinderpal](https://github.com/Arvinderpal))",
-    "author": "Arvinderpal",
-    "author_url": "https://github.com/Arvinderpal",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85524",
-    "pr_number": 85524,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.10"
-  },
-  "85542": {
-    "commit": "541b4e4b395b5124f3517b4e62f3dc604016db11",
-    "text": "e2e-test-framework: add e2e test namespace dump if all tests succeed but the cleanup fails.",
-    "markdown": "e2e-test-framework: add e2e test namespace dump if all tests succeed but the cleanup fails. ([#85542](https://github.com/kubernetes/kubernetes/pull/85542), [@schrodit](https://github.com/schrodit))",
-    "author": "schrodit",
-    "author_url": "https://github.com/schrodit",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85542",
-    "pr_number": 85542,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["bug", "failing-test"],
-    "sigs": ["testing"],
-    "release_version": "1.14.10"
-  },
-  "85543": {
-    "commit": "650c797d3faedcbd3ad60cad065cabf305bb49a1",
-    "text": "addons: elasticsearch discovery supports IPv6",
-    "markdown": "addons: elasticsearch discovery supports IPv6 ([#85543](https://github.com/kubernetes/kubernetes/pull/85543), [@SataQiu](https://github.com/SataQiu))",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85543",
-    "pr_number": 85543,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85571": {
-    "commit": "b42f21982c965630f82f053425ba067d818203f5",
-    "text": "kubectl/drain: add disable-eviction option.\nForce drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, and should be used with caution.",
-    "markdown": "kubectl/drain: add disable-eviction option.\n  Force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, and should be used with caution. ([#85571](https://github.com/kubernetes/kubernetes/pull/85571), [@michaelgugino](https://github.com/michaelgugino))\n\n  Courtesy of SIG CLI",
-    "author": "michaelgugino",
-    "author_url": "https://github.com/michaelgugino",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85571",
-    "pr_number": 85571,
-    "areas": ["kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "85577": {
-    "commit": "93eb2e38a7800087f0ab2ae058cccd4dcceae8f9",
-    "text": "kubectl/drain: add skip-wait-for-delete-timeout option.\nIf pod DeletionTimestamp older than N seconds, skip waiting for the pod.  Seconds must be greater than 0 to skip.",
-    "markdown": "kubectl/drain: add skip-wait-for-delete-timeout option.\n  If pod DeletionTimestamp older than N seconds, skip waiting for the pod.  Seconds must be greater than 0 to skip. ([#85577](https://github.com/kubernetes/kubernetes/pull/85577), [@michaelgugino](https://github.com/michaelgugino))\n\n  Courtesy of SIG CLI",
-    "documentation": [
-      {
-        "description": "[Other doc]",
-        "url": "https://github.com/kubernetes/enhancements/issues/1339",
-        "type": "KEP"
-      }
-    ],
-    "author": "michaelgugino",
-    "author_url": "https://github.com/michaelgugino",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85577",
-    "pr_number": 85577,
-    "areas": ["kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "85603": {
-    "commit": "e51caab73241b5c1598dfac5acbb2bde291a5e1f",
-    "text": "kubeadm: make sure images are pre-pulled even if a tag did not change but their contents changed",
-    "markdown": "kubeadm: make sure images are pre-pulled even if a tag did not change but their contents changed ([#85603](https://github.com/kubernetes/kubernetes/pull/85603), [@bart0sh](https://github.com/bart0sh))",
-    "author": "bart0sh",
-    "author_url": "https://github.com/bart0sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85603",
-    "pr_number": 85603,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.10"
-  },
-  "85609": {
-    "commit": "7907c63fad1ccb7905d2d27c643387046ab0f4ad",
-    "text": "kubeadm now supports automatic calculations of dual-stack node cidr masks to kube-controller-manager.",
-    "markdown": "kubeadm now supports automatic calculations of dual-stack node cidr masks to kube-controller-manager. ([#85609](https://github.com/kubernetes/kubernetes/pull/85609), [@Arvinderpal](https://github.com/Arvinderpal))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "Arvinderpal",
-    "author_url": "https://github.com/Arvinderpal",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85609",
-    "pr_number": 85609,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.10"
-  },
-  "85610": {
-    "commit": "82ee37f3e1ef2be3bfca8031e9babfc748084b4f",
-    "text": "Update Cluster Autoscaler to 1.17.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.17.0",
-    "markdown": "Update Cluster Autoscaler to 1.17.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.17.0 ([#85610](https://github.com/kubernetes/kubernetes/pull/85610), [@losipiuk](https://github.com/losipiuk))",
-    "author": "losipiuk",
-    "author_url": "https://github.com/losipiuk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85610",
-    "pr_number": 85610,
-    "kinds": ["bug"],
-    "sigs": ["autoscaling", "cluster-lifecycle"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85621": {
-    "commit": "eef4c00ae93bd51125c215918a7b1c632d298610",
-    "text": "None.",
-    "markdown": "None. ([#85621](https://github.com/kubernetes/kubernetes/pull/85621), [@chewong](https://github.com/chewong))",
-    "author": "chewong",
-    "author_url": "https://github.com/chewong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85621",
-    "pr_number": 85621,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85639": {
-    "commit": "674695c544296c90e0a4fa0abdab71b3679a82c3",
-    "text": "kubeadm now errors out whenever a not supported component config version is supplied for the kubelet and kube-proxy",
-    "markdown": "kubeadm now errors out whenever a not supported component config version is supplied for the kubelet and kube-proxy ([#85639](https://github.com/kubernetes/kubernetes/pull/85639), [@rosti](https://github.com/rosti))",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-cluster-lifecycle/kubeadm/20190925-component-configs.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85639",
-    "pr_number": 85639,
-    "areas": ["kubeadm"],
-    "kinds": ["design"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.10"
-  },
-  "85675": {
-    "commit": "b920b388e90d32ea268b1b23142b65514fe6139e",
-    "text": "Fixed \"requested device X but found Y\" attach error on AWS.",
-    "markdown": "Fixed \"requested device X but found Y\" attach error on AWS. ([#85675](https://github.com/kubernetes/kubernetes/pull/85675), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85675",
-    "pr_number": 85675,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85722": {
-    "commit": "1d343c8a0ce3e4d4845f7c34d06a1e2f25c975b7",
-    "text": "Filter published OpenAPI schema by making nullable, required fields non-required in order to avoid kubectl to wrongly reject null values.",
-    "markdown": "Filter published OpenAPI schema by making nullable, required fields non-required in order to avoid kubectl to wrongly reject null values. ([#85722](https://github.com/kubernetes/kubernetes/pull/85722), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85722",
-    "pr_number": 85722,
-    "areas": ["custom-resources"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.10"
-  },
-  "85739": {
-    "commit": "220e2ceba0784fe230d590d58e26d87cbf05af13",
-    "text": "Update go-winio module version from 0.4.11 to 0.4.14",
-    "markdown": "Update go-winio module version from 0.4.11 to 0.4.14 ([#85739](https://github.com/kubernetes/kubernetes/pull/85739), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85739",
-    "pr_number": 85739,
-    "areas": ["dependency"],
-    "kinds": ["bug"],
-    "sigs": ["windows"],
-    "release_version": "1.14.10"
-  },
-  "85745": {
-    "commit": "3769de9401474ea22c0ddbacde896911ddd87700",
-    "text": "kubeadm: uses the apiserver AdvertiseAddress IP family to choose the etcd endpoint IP family for non external etcd clusters",
-    "markdown": "kubeadm: uses the apiserver AdvertiseAddress IP family to choose the etcd endpoint IP family for non external etcd clusters ([#85745](https://github.com/kubernetes/kubernetes/pull/85745), [@aojea](https://github.com/aojea))",
-    "author": "aojea",
-    "author_url": "https://github.com/aojea",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85745",
-    "pr_number": 85745,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.10"
-  },
-  "85763": {
-    "commit": "1ca289dfc11547af7b1e7aa6c6fd09972260a7c6",
-    "text": "kubeadm: retry `kubeadm-config` ConfigMap creation or mutation if the apiserver is not responding. This will improve resiliency when joining new control plane nodes.",
-    "markdown": "kubeadm: retry `kubeadm-config` ConfigMap creation or mutation if the apiserver is not responding. This will improve resiliency when joining new control plane nodes. ([#85763](https://github.com/kubernetes/kubernetes/pull/85763), [@ereslibre](https://github.com/ereslibre))",
-    "author": "ereslibre",
-    "author_url": "https://github.com/ereslibre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85763",
-    "pr_number": 85763,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.10"
-  },
-  "85793": {
-    "commit": "85b5a046f7de104281225485797232d6cdd71389",
-    "text": "Fixed issue with addon-resizer using deprecated extensions APIs",
-    "markdown": "Fixed issue with addon-resizer using deprecated extensions APIs ([#85793](https://github.com/kubernetes/kubernetes/pull/85793), [@bskiba](https://github.com/bskiba))",
-    "author": "bskiba",
-    "author_url": "https://github.com/bskiba",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85793",
-    "pr_number": 85793,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85810": {
-    "commit": "19ee1ea9fa61b2a3fe4d86438bef5203bfe1a756",
-    "text": "Resolved regression in admission, authentication, and authorization webhook performance in v1.17.0-rc.1",
-    "markdown": "Resolved regression in admission, authentication, and authorization webhook performance in v1.17.0-rc.1 ([#85810](https://github.com/kubernetes/kubernetes/pull/85810), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85810",
-    "pr_number": 85810,
-    "areas": ["apiserver", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85817": {
-    "commit": "6e0ce9fb2ff8f26d1becafedfc732d603363b9a7",
-    "text": "kubeadm: Forward cluster name to the controller-manager arguments",
-    "markdown": "kubeadm: Forward cluster name to the controller-manager arguments ([#85817](https://github.com/kubernetes/kubernetes/pull/85817), [@ereslibre](https://github.com/ereslibre))",
-    "author": "ereslibre",
-    "author_url": "https://github.com/ereslibre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85817",
-    "pr_number": 85817,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.10"
-  },
-  "85857": {
-    "commit": "2fbe432d2347d7f808054f92f2b146e8a7dd2de8",
-    "text": "During initialization phase (preflight), kubeadm now verifies the presence of the conntrack executable",
-    "markdown": "During initialization phase (preflight), kubeadm now verifies the presence of the conntrack executable ([#85857](https://github.com/kubernetes/kubernetes/pull/85857), [@hnanni](https://github.com/hnanni))",
-    "author": "hnanni",
-    "author_url": "https://github.com/hnanni",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85857",
-    "pr_number": 85857,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.10"
-  },
-  "85874": {
-    "commit": "398e2bcc734f04686ff89fdd9f5b47cf12cd525b",
-    "text": "- Renamed Kubelet metric certificate_manager_server_expiration_seconds to certificate_manager_server_ttl_seconds and changed to report the second until expiration at read time rather than absolute time of expiry.\n- Improved accuracy of Kubelet metric rest_client_exec_plugin_ttl_seconds.",
-    "markdown": "- Renamed Kubelet metric certificate_manager_server_expiration_seconds to certificate_manager_server_ttl_seconds and changed to report the second until expiration at read time rather than absolute time of expiry.\n  - Improved accuracy of Kubelet metric rest_client_exec_plugin_ttl_seconds. ([#85874](https://github.com/kubernetes/kubernetes/pull/85874), [@sambdavidson](https://github.com/sambdavidson))",
-    "author": "sambdavidson",
-    "author_url": "https://github.com/sambdavidson",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85874",
-    "pr_number": 85874,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "auth", "cluster-lifecycle", "instrumentation", "node"],
-    "duplicate": true,
-    "release_version": "1.14.10"
-  },
-  "85885": {
-    "commit": "50437b4c5da29baba1f8aeef201c2b7736cd42d2",
-    "text": "VMSS cache is added so that less chances of VMSS GET throttling",
-    "markdown": "VMSS cache is added so that less chances of VMSS GET throttling ([#85885](https://github.com/kubernetes/kubernetes/pull/85885), [@nilo19](https://github.com/nilo19))",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85885",
-    "pr_number": 85885,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.10"
-  },
-  "85990": {
-    "commit": "b5040be796b71f07de2f0be782d14e9b24b018b7",
-    "text": "Fix LoadBalancer rule checking so that no unexpected LoadBalancer updates are made",
-    "markdown": "Fix LoadBalancer rule checking so that no unexpected LoadBalancer updates are made ([#85990](https://github.com/kubernetes/kubernetes/pull/85990), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85990",
-    "pr_number": 85990,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.10"
-  },
-  "86022": {
-    "commit": "d842c194c21f86ddcecae303ce9d8dcce882a18c",
-    "text": "Fixed an issue that the scheduler only returns the first failure reason.",
-    "markdown": "Fixed an issue that the scheduler only returns the first failure reason. ([#86022](https://github.com/kubernetes/kubernetes/pull/86022), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86022",
-    "pr_number": 86022,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.10"
-  },
-  "86078": {
-    "commit": "939da5e9cf7a041fd5cdf992449372e7b9833162",
-    "text": "Make error message and service event message more clear",
-    "markdown": "Make error message and service event message more clear ([#86078](https://github.com/kubernetes/kubernetes/pull/86078), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86078",
-    "pr_number": 86078,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.10"
-  }
-}
diff --git a/src/assets/release-notes-1.14.2.json b/src/assets/release-notes-1.14.2.json
deleted file mode 100644
index 89b974b8..00000000
--- a/src/assets/release-notes-1.14.2.json
+++ /dev/null
@@ -1,1614 +0,0 @@
-{
-  "60771": {
-    "commit": "d1e7bb297c02bbbcb46dd7eeebae41cd042bb48c",
-    "text": "Fixes a bug where the kube-apiserver won't start if a broken webhook is installed and the client ca configmap is empty",
-    "markdown": "Fixes a bug where the kube-apiserver won't start if a broken webhook is installed and the client ca configmap is empty ([#60771](https://github.com/kubernetes/kubernetes/pull/60771), [@jennybuckley](https://github.com/jennybuckley))",
-    "author": "jennybuckley",
-    "author_url": "https://github.com/jennybuckley",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/60771",
-    "pr_number": 60771,
-    "areas": ["apiserver", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "71735": {
-    "commit": "2490e035d79aa01665597aae972fb6672ef74756",
-    "text": "The userspace proxy now respects the IPTables proxy's minSyncInterval parameter.",
-    "markdown": "The userspace proxy now respects the IPTables proxy's minSyncInterval parameter. ([#71735](https://github.com/kubernetes/kubernetes/pull/71735), [@dcbw](https://github.com/dcbw))",
-    "author": "dcbw",
-    "author_url": "https://github.com/dcbw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/71735",
-    "pr_number": 71735,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.2"
-  },
-  "73101": {
-    "commit": "aa84b99fa0a49d0206984f4deb5ad3b58817f838",
-    "text": "util/initsystem: add support for the OpenRC init system",
-    "markdown": "util/initsystem: add support for the OpenRC init system ([#73101](https://github.com/kubernetes/kubernetes/pull/73101), [@oz123](https://github.com/oz123))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Node",
-    "author": "oz123",
-    "author_url": "https://github.com/oz123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73101",
-    "pr_number": 73101,
-    "areas": ["kubeadm", "kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "node"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "73664": {
-    "commit": "e5dd4521a8c938604287684109b22129b5395a3e",
-    "text": "kubectl exec now allows using resource name (e.g., deployment/mydeployment) to select a matching pod.\nkubectl exec now allows using --pod-running-timeout flag to wait till at least one pod is running.",
-    "markdown": "kubectl exec now allows using resource name (e.g., deployment/mydeployment) to select a matching pod.\n  kubectl exec now allows using --pod-running-timeout flag to wait till at least one pod is running. ([#73664](https://github.com/kubernetes/kubernetes/pull/73664), [@prksu](https://github.com/prksu))\n\n  Courtesy of SIG CLI, and SIG Testing",
-    "author": "prksu",
-    "author_url": "https://github.com/prksu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73664",
-    "pr_number": 73664,
-    "areas": ["kubectl", "test"],
-    "kinds": ["feature"],
-    "sigs": ["cli", "testing"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "73795": {
-    "commit": "6a48257627d3bce70457342cb7514f36b5240dc5",
-    "text": "Add RuntimeClass restrictions \u0026 defaulting to PodSecurityPolicy.",
-    "markdown": "Add RuntimeClass restrictions \u0026 defaulting to PodSecurityPolicy. ([#73795](https://github.com/kubernetes/kubernetes/pull/73795), [@tallclair](https://github.com/tallclair))\n\n  Courtesy of SIG Apps, SIG Architecture, SIG Auth, and SIG Node",
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73795",
-    "pr_number": 73795,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["apps", "architecture", "auth", "node"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "74074": {
-    "commit": "107595eb5e2da6f79a07fb8b5900deec7f063141",
-    "text": "Introduce API for watch bookmark events.\nIntroduce Alpha field `AllowWatchBookmarks` in ListOptions for requesting watch bookmarks from apiserver. The implementation in apiserver is hidden behind feature gate `WatchBookmark` (currently in Alpha stage).",
-    "markdown": "Introduce API for watch bookmark events.\n  Introduce Alpha field `AllowWatchBookmarks` in ListOptions for requesting watch bookmarks from apiserver. The implementation in apiserver is hidden behind feature gate `WatchBookmark` (currently in Alpha stage). ([#74074](https://github.com/kubernetes/kubernetes/pull/74074), [@wojtek-t](https://github.com/wojtek-t))",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74074",
-    "pr_number": 74074,
-    "areas": ["apiserver"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "scalability"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "74610": {
-    "commit": "98b6f359992e7d7e6de77aa25ba3315586a8f6b2",
-    "text": "StorageObjectInUseProtection admission plugin is additionally enabled by default.\nSo default enabled admission plugins are now `NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,StorageObjectInUseProtection`. Please note that if you previously had not set the `--admission-control` flag, your cluster behavior may change (to be more standard).",
-    "markdown": "StorageObjectInUseProtection admission plugin is additionally enabled by default.\n  So default enabled admission plugins are now `NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,StorageObjectInUseProtection`. Please note that if you previously had not set the `--admission-control` flag, your cluster behavior may change (to be more standard). ([#74610](https://github.com/kubernetes/kubernetes/pull/74610), [@oomichi](https://github.com/oomichi))",
-    "author": "oomichi",
-    "author_url": "https://github.com/oomichi",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74610",
-    "pr_number": 74610,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "74690": {
-    "commit": "dc2c9e8bd522d9afe19a1da7e3aa56dca450a024",
-    "text": "Override protocol between etcd server and kube-apiserver on master with HTTPS instead HTTP when mTLS is enabled in GCE",
-    "markdown": "Override protocol between etcd server and kube-apiserver on master with HTTPS instead HTTP when mTLS is enabled in GCE ([#74690](https://github.com/kubernetes/kubernetes/pull/74690), [@wenjiaswe](https://github.com/wenjiaswe))",
-    "author": "wenjiaswe",
-    "author_url": "https://github.com/wenjiaswe",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74690",
-    "pr_number": 74690,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "74840": {
-    "commit": "fa833a1e33d45b6438b6c2afa45ff252c5aa582b",
-    "text": "Packets considered INVALID by conntrack are now dropped. In particular, this fixes\na problem where spurious retransmits in a long-running TCP connection to a service\nIP could result in the connection being closed with the error \"Connection reset by\npeer\"",
-    "markdown": "Packets considered INVALID by conntrack are now dropped. In particular, this fixes\n  a problem where spurious retransmits in a long-running TCP connection to a service\n  IP could result in the connection being closed with the error \"Connection reset by\n  peer\" ([#74840](https://github.com/kubernetes/kubernetes/pull/74840), [@anfernee](https://github.com/anfernee))",
-    "author": "anfernee",
-    "author_url": "https://github.com/anfernee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74840",
-    "pr_number": 74840,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.2"
-  },
-  "74855": {
-    "commit": "eb65eacfcab4b53eb65953b9f59a00e3e2b5a909",
-    "text": "- Added port configuration to Admission webhook configuration service reference.\n- Added port configuration to AuditSink webhook configuration service reference.\n- Added port configuration to CRD Conversion webhook configuration service reference.\n- Added port configuration to kube-aggregator service reference.",
-    "markdown": "- Added port configuration to Admission webhook configuration service reference.\n  - Added port configuration to AuditSink webhook configuration service reference.\n  - Added port configuration to CRD Conversion webhook configuration service reference.\n  - Added port configuration to kube-aggregator service reference. ([#74855](https://github.com/kubernetes/kubernetes/pull/74855), [@mbohlool](https://github.com/mbohlool))",
-    "author": "mbohlool",
-    "author_url": "https://github.com/mbohlool",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74855",
-    "pr_number": 74855,
-    "areas": [
-      "admission-control",
-      "apiserver",
-      "custom-resources",
-      "dependency",
-      "release-eng",
-      "test"
-    ],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "auth", "release", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "74856": {
-    "commit": "d5245b9563d1c3e9c67bc91ca1aa0d0210da2e13",
-    "text": "Remove extra pod creation expections when daemonset fails to create pods in batches.",
-    "markdown": "Remove extra pod creation expections when daemonset fails to create pods in batches. ([#74856](https://github.com/kubernetes/kubernetes/pull/74856), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74856",
-    "pr_number": 74856,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "74910": {
-    "commit": "4f08ea91df235394374a616b9f6a7e3511db0705",
-    "text": "Add TLS termination support for NLB",
-    "markdown": "Add TLS termination support for NLB ([#74910](https://github.com/kubernetes/kubernetes/pull/74910), [@M00nF1sh](https://github.com/M00nF1sh))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "M00nF1sh",
-    "author_url": "https://github.com/M00nF1sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74910",
-    "pr_number": 74910,
-    "areas": ["cloudprovider"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "74942": {
-    "commit": "8ec6167f61463cbc6874514b02d0137d51b23542",
-    "text": "Migrate oom watcher not relying on cAdviosr's API any more",
-    "markdown": "Migrate oom watcher not relying on cAdviosr's API any more ([#74942](https://github.com/kubernetes/kubernetes/pull/74942), [@WanLinghao](https://github.com/WanLinghao))\n\n  Courtesy of SIG Node",
-    "author": "WanLinghao",
-    "author_url": "https://github.com/WanLinghao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74942",
-    "pr_number": 74942,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "75278": {
-    "commit": "097d58523a095cd1b2a4dc44273eb46234740a98",
-    "text": "UpdateContainerResources is no longer recorded as a `container_status` operation. It now uses the label `update_container`",
-    "markdown": "UpdateContainerResources is no longer recorded as a `container_status` operation. It now uses the label `update_container` ([#75278](https://github.com/kubernetes/kubernetes/pull/75278), [@Nessex](https://github.com/Nessex))",
-    "author": "Nessex",
-    "author_url": "https://github.com/Nessex",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75278",
-    "pr_number": 75278,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.14.2"
-  },
-  "75501": {
-    "commit": "a93f803f8e400f1d42dc812bc51932ff3b31798a",
-    "text": "scheduler: add metrics to record number of pending pods in different queues",
-    "markdown": "scheduler: add metrics to record number of pending pods in different queues ([#75501](https://github.com/kubernetes/kubernetes/pull/75501), [@Huang-Wei](https://github.com/Huang-Wei))\n\n  Courtesy of SIG Apps, and SIG Scheduling",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75501",
-    "pr_number": 75501,
-    "kinds": ["feature"],
-    "sigs": ["apps", "scheduling"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "75658": {
-    "commit": "e3c3f75b5b3d1b236ecc528777cfc1ec9b3cf112",
-    "text": "Update cri-tools to v1.14.0",
-    "markdown": "Update cri-tools to v1.14.0 ([#75658](https://github.com/kubernetes/kubernetes/pull/75658), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75658",
-    "pr_number": 75658,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.2"
-  },
-  "75848": {
-    "commit": "cf22c56c7fbeaa166e5e86ebc4c0b75250b9843a",
-    "text": "Implement and update interfaces and skeleton for the scheduling framework.",
-    "markdown": "Implement and update interfaces and skeleton for the scheduling framework. ([#75848](https://github.com/kubernetes/kubernetes/pull/75848), [@bsalamat](https://github.com/bsalamat))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "bsalamat",
-    "author_url": "https://github.com/bsalamat",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75848",
-    "pr_number": 75848,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "75849": {
-    "commit": "d5dbc0019123f42c678a1f012a068941d56087c8",
-    "text": "Add ListPager.EachListItem utility function to client-go to enable incremental processing of chunked list responses",
-    "markdown": "Add ListPager.EachListItem utility function to client-go to enable incremental processing of chunked list responses ([#75849](https://github.com/kubernetes/kubernetes/pull/75849), [@jpbetz](https://github.com/jpbetz))\n\n  Courtesy of SIG API Machinery, and SIG Apps",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75849",
-    "pr_number": 75849,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "apps"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "75853": {
-    "commit": "f8d0b21b980f33e218b7e1f01a13bf3bd90713f9",
-    "text": "Use stdlib to log stack trace when a panic occurs",
-    "markdown": "Use stdlib to log stack trace when a panic occurs ([#75853](https://github.com/kubernetes/kubernetes/pull/75853), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75853",
-    "pr_number": 75853,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.2"
-  },
-  "75986": {
-    "commit": "88992ba185c1c9205eef93ed9c558a70bd9da641",
-    "text": "Reduce event spam for AttachVolume storage operation",
-    "markdown": "Reduce event spam for AttachVolume storage operation ([#75986](https://github.com/kubernetes/kubernetes/pull/75986), [@mucahitkurt](https://github.com/mucahitkurt))",
-    "author": "mucahitkurt",
-    "author_url": "https://github.com/mucahitkurt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75986",
-    "pr_number": 75986,
-    "kinds": ["cleanup"],
-    "sigs": ["storage"],
-    "release_version": "1.14.2"
-  },
-  "75987": {
-    "commit": "8136cfc512754e0b3a943acc2a0d12aa3903e019",
-    "text": "The `series.state` field in the events.k8s.io/v1beta1 Event API is deprecated and will be removed in v1.18",
-    "markdown": "The `series.state` field in the events.k8s.io/v1beta1 Event API is deprecated and will be removed in v1.18 ([#75987](https://github.com/kubernetes/kubernetes/pull/75987), [@yastij](https://github.com/yastij))",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75987",
-    "pr_number": 75987,
-    "kinds": ["api-change"],
-    "sigs": ["apps", "instrumentation", "scalability"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "75993": {
-    "commit": "1fb8ed12a2d1d50a890f24c67c5fc0d37d4ffb04",
-    "text": "Adding ListMeta.RemainingItemCount. When responding a LIST request, if the server has more data available, and if the request does not contain label selectors or field selectors, the server sets the ListOptions.RemainingItemCount to the number of remaining objects.",
-    "markdown": "Adding ListMeta.RemainingItemCount. When responding a LIST request, if the server has more data available, and if the request does not contain label selectors or field selectors, the server sets the ListOptions.RemainingItemCount to the number of remaining objects. ([#75993](https://github.com/kubernetes/kubernetes/pull/75993), [@caesarxuchao](https://github.com/caesarxuchao))",
-    "author": "caesarxuchao",
-    "author_url": "https://github.com/caesarxuchao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75993",
-    "pr_number": 75993,
-    "areas": ["apiserver", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "76014": {
-    "commit": "4c4c378d99c207f026a3e9b2c1fac625ce613535",
-    "text": "improve `kubectl auth can-i` command by warning users when they try access resource out of scope",
-    "markdown": "improve `kubectl auth can-i` command by warning users when they try access resource out of scope ([#76014](https://github.com/kubernetes/kubernetes/pull/76014), [@WanLinghao](https://github.com/WanLinghao))",
-    "author": "WanLinghao",
-    "author_url": "https://github.com/WanLinghao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76014",
-    "pr_number": 76014,
-    "areas": ["kubectl", "test"],
-    "kinds": ["bug"],
-    "sigs": ["auth", "cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "76060": {
-    "commit": "8f8d9e2c726e98225123c032462f3fd155d9bb6e",
-    "text": "Fixes bug in DaemonSetController causing it to stop processing some DaemonSets for 5 minutes after node removal.",
-    "markdown": "Fixes bug in DaemonSetController causing it to stop processing some DaemonSets for 5 minutes after node removal. ([#76060](https://github.com/kubernetes/kubernetes/pull/76060), [@krzysztof-jastrzebski](https://github.com/krzysztof-jastrzebski))",
-    "author": "krzysztof-jastrzebski",
-    "author_url": "https://github.com/krzysztof-jastrzebski",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76060",
-    "pr_number": 76060,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.14.2"
-  },
-  "76062": {
-    "commit": "3409a5362aac664594f7e87e23576b230b400730",
-    "text": "Create a new `kubectl rollout restart` command that does a rolling restart of a deployment.",
-    "markdown": "Create a new `kubectl rollout restart` command that does a rolling restart of a deployment. ([#76062](https://github.com/kubernetes/kubernetes/pull/76062), [@apelisse](https://github.com/apelisse))\n\n  Courtesy of SIG CLI, and SIG Testing",
-    "author": "apelisse",
-    "author_url": "https://github.com/apelisse",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76062",
-    "pr_number": 76062,
-    "areas": ["kubectl", "test"],
-    "kinds": ["feature"],
-    "sigs": ["cli", "testing"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "76074": {
-    "commit": "b7858e31d416312aab394df7c0d0c42e2c179dce",
-    "text": "Change kubelet probe metrics to counter type.\nThe metrics `prober_probe_result` is replaced by `prober_probe_total`.",
-    "markdown": "Change kubelet probe metrics to counter type.\n  The metrics `prober_probe_result` is replaced by `prober_probe_total`. ([#76074](https://github.com/kubernetes/kubernetes/pull/76074), [@danielqsj](https://github.com/danielqsj))\n\n  Courtesy of SIG Instrumentation, and SIG Node",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76074",
-    "pr_number": 76074,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["instrumentation", "node"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "76161": {
-    "commit": "c082ace102e217846833baace01d80966369ecd8",
-    "text": "`kubectl get -w` now prints custom resource definitions with custom print columns",
-    "markdown": "`kubectl get -w` now prints custom resource definitions with custom print columns ([#76161](https://github.com/kubernetes/kubernetes/pull/76161), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG CLI",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76161",
-    "pr_number": 76161,
-    "areas": ["kubectl"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "76221": {
-    "commit": "922f3603eae623d7f0e56fee4bf5b05eb7e82c43",
-    "text": "As of Kubernetes 1.15, the SupportNodePidsLimit feature introduced as alpha in Kubernetes 1.14 is now beta, and the ability to utilize it is enabled by default.  It is no longer necessary to set the feature gate `SupportNodePidsLimit=true`.  In all other respects, this functionality behaves as it did in Kubernetes 1.14.",
-    "markdown": "As of Kubernetes 1.15, the SupportNodePidsLimit feature introduced as alpha in Kubernetes 1.14 is now beta, and the ability to utilize it is enabled by default.  It is no longer necessary to set the feature gate `SupportNodePidsLimit=true`.  In all other respects, this functionality behaves as it did in Kubernetes 1.14. ([#76221](https://github.com/kubernetes/kubernetes/pull/76221), [@RobertKrawitz](https://github.com/RobertKrawitz))\n\n  Courtesy of SIG Testing",
-    "author": "RobertKrawitz",
-    "author_url": "https://github.com/RobertKrawitz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76221",
-    "pr_number": 76221,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["testing"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "76238": {
-    "commit": "888b81b638171929d2a7a032bba7386a1cbff9c7",
-    "text": "Default TTL for DNS records in kubernetes zone is changed from 5s to 30s to keep consistent with old dnsmasq based kube-dns. The TTL can be customized with command `kubectl edit -n kube-system configmap/coredns`.",
-    "markdown": "Default TTL for DNS records in kubernetes zone is changed from 5s to 30s to keep consistent with old dnsmasq based kube-dns. The TTL can be customized with command `kubectl edit -n kube-system configmap/coredns`. ([#76238](https://github.com/kubernetes/kubernetes/pull/76238), [@Dieken](https://github.com/Dieken))",
-    "author": "Dieken",
-    "author_url": "https://github.com/Dieken",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76238",
-    "pr_number": 76238,
-    "areas": ["kubeadm", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "76243": {
-    "commit": "8a9ed4c91a54b1b29c3b60dad815c372bcfdef76",
-    "text": "2X performance improvement on both required and preferred PodAffinity.",
-    "markdown": "2X performance improvement on both required and preferred PodAffinity. ([#76243](https://github.com/kubernetes/kubernetes/pull/76243), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76243",
-    "pr_number": 76243,
-    "kinds": ["design"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.2"
-  },
-  "76320": {
-    "commit": "4dc05dd9dff1283984acd77d68d8dce7b7cdfb68",
-    "text": "kube-proxy: HealthzBindAddress and MetricsBindAddress support ipv6 address.",
-    "markdown": "kube-proxy: HealthzBindAddress and MetricsBindAddress support ipv6 address. ([#76320](https://github.com/kubernetes/kubernetes/pull/76320), [@JieJhih](https://github.com/JieJhih))",
-    "author": "JieJhih",
-    "author_url": "https://github.com/JieJhih",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76320",
-    "pr_number": 76320,
-    "kinds": ["cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.14.2"
-  },
-  "76327": {
-    "commit": "7defecbc2c88f874eb0cc4dd8a2a2911cc8e116a",
-    "text": "kubeadm: disable the kube-proxy DaemonSet on non-Linux nodes. This step is required to support Windows worker nodes.",
-    "markdown": "kubeadm: disable the kube-proxy DaemonSet on non-Linux nodes. This step is required to support Windows worker nodes. ([#76327](https://github.com/kubernetes/kubernetes/pull/76327), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76327",
-    "pr_number": 76327,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "76341": {
-    "commit": "d424bedc9b7caa4f153c422299411e76b5e64986",
-    "text": "Fix issue in Portworx volume driver causing controller manager to crash",
-    "markdown": "Fix issue in Portworx volume driver causing controller manager to crash ([#76341](https://github.com/kubernetes/kubernetes/pull/76341), [@harsh-px](https://github.com/harsh-px))",
-    "author": "harsh-px",
-    "author_url": "https://github.com/harsh-px",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76341",
-    "pr_number": 76341,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.2"
-  },
-  "76352": {
-    "commit": "ae8a4fcb0d79982c1c8f48be28c34ca7a2ebd6c0",
-    "text": "Reduce GCE log rotation check from 1 hour to every 5 minutes.  Rotation policy is unchanged (new day starts, log file size \u003e 100MB).",
-    "markdown": "Reduce GCE log rotation check from 1 hour to every 5 minutes.  Rotation policy is unchanged (new day starts, log file size \u003e 100MB). ([#76352](https://github.com/kubernetes/kubernetes/pull/76352), [@jpbetz](https://github.com/jpbetz))",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76352",
-    "pr_number": 76352,
-    "areas": ["platform/gce"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.2"
-  },
-  "76353": {
-    "commit": "134ed2e7031bc91c7b35796553fbcd3ba8dc042f",
-    "text": "Added `CNI_VERSION` and `CNI_SHA1` environment variables in kube-up.sh to configure CNI versions on GCE.",
-    "markdown": "Added `CNI_VERSION` and `CNI_SHA1` environment variables in kube-up.sh to configure CNI versions on GCE. ([#76353](https://github.com/kubernetes/kubernetes/pull/76353), [@Random-Liu](https://github.com/Random-Liu))\n\n  Courtesy of SIG Network, and SIG Node",
-    "author": "Random-Liu",
-    "author_url": "https://github.com/Random-Liu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76353",
-    "pr_number": 76353,
-    "kinds": ["feature"],
-    "sigs": ["network", "node"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "76390": {
-    "commit": "3148eb750da9f2259c4105e80a0088acdf1f40cd",
-    "text": "kubeadm: add optional ECDSA support.\nkubeadm still generates RSA keys when deploying a node, but also accepts ECDSA\nkeys if they exist already in the directory specified in --cert-dir option.",
-    "markdown": "kubeadm: add optional ECDSA support.\n  kubeadm still generates RSA keys when deploying a node, but also accepts ECDSA\n  keys if they exist already in the directory specified in --cert-dir option. ([#76390](https://github.com/kubernetes/kubernetes/pull/76390), [@rojkov](https://github.com/rojkov))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "rojkov",
-    "author_url": "https://github.com/rojkov",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76390",
-    "pr_number": 76390,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "76437": {
-    "commit": "0da72a1ef3a3edf752e931681c171f09d545a79b",
-    "text": "Bump metrics-server to v0.3.2",
-    "markdown": "Bump metrics-server to v0.3.2 ([#76437](https://github.com/kubernetes/kubernetes/pull/76437), [@brett-elliott](https://github.com/brett-elliott))",
-    "author": "brett-elliott",
-    "author_url": "https://github.com/brett-elliott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76437",
-    "pr_number": 76437,
-    "kinds": ["failing-test"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.2"
-  },
-  "76474": {
-    "commit": "17fe18bd9cc5652a5dee688255323ce2d07538dd",
-    "text": "Updated klog to 0.3.0",
-    "markdown": "Updated klog to 0.3.0 ([#76474](https://github.com/kubernetes/kubernetes/pull/76474), [@vincepri](https://github.com/vincepri))",
-    "author": "vincepri",
-    "author_url": "https://github.com/vincepri",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76474",
-    "pr_number": 76474,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "node"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "76505": {
-    "commit": "299aa5d7a62188e27d7a6cd8287b2411574587ba",
-    "text": "kubeadm: Fix a bug where if couple of CRIs are installed a user override of the CRI during join (via kubeadm join --cri-socket ...) is ignored and kubeadm bails out with an error",
-    "markdown": "kubeadm: Fix a bug where if couple of CRIs are installed a user override of the CRI during join (via kubeadm join --cri-socket ...) is ignored and kubeadm bails out with an error ([#76505](https://github.com/kubernetes/kubernetes/pull/76505), [@rosti](https://github.com/rosti))",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76505",
-    "pr_number": 76505,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.2"
-  },
-  "76573": {
-    "commit": "64a0441c2b43a13da0c3e614706e99c52392c449",
-    "text": "fix detach azure disk back off issue which has too big lock in failure retry condition",
-    "markdown": "fix detach azure disk back off issue which has too big lock in failure retry condition ([#76573](https://github.com/kubernetes/kubernetes/pull/76573), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76573",
-    "pr_number": 76573,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "76576": {
-    "commit": "b3847120241251d624d570e2212937c659614ea7",
-    "text": "Update to use go 1.12.4",
-    "markdown": "Update to use go 1.12.4 ([#76576](https://github.com/kubernetes/kubernetes/pull/76576), [@cblecker](https://github.com/cblecker))\n\n  Courtesy of SIG Release, and SIG Testing",
-    "author": "cblecker",
-    "author_url": "https://github.com/cblecker",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76576",
-    "pr_number": 76576,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["release", "testing"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "76595": {
-    "commit": "f44d04c1a205a4d650ec7ba3ec9a3a0ae0902366",
-    "text": "Remove the function Parallelize, please convert to use the function ParallelizeUntil.",
-    "markdown": "Remove the function Parallelize, please convert to use the function ParallelizeUntil. ([#76595](https://github.com/kubernetes/kubernetes/pull/76595), [@danielqsj](https://github.com/danielqsj))",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76595",
-    "pr_number": 76595,
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "76603": {
-    "commit": "60a4c3c0030a5217a597902f05f587d47cd6eff8",
-    "text": "Fix TestEventChannelFull random fail",
-    "markdown": "Fix TestEventChannelFull random fail ([#76603](https://github.com/kubernetes/kubernetes/pull/76603), [@changyaowei](https://github.com/changyaowei))",
-    "author": "changyaowei",
-    "author_url": "https://github.com/changyaowei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76603",
-    "pr_number": 76603,
-    "areas": ["kubelet"],
-    "kinds": ["flake"],
-    "sigs": ["node"],
-    "release_version": "1.14.2"
-  },
-  "76627": {
-    "commit": "e1fa195451cf7d21e6ef080fe5bd046fb8d4fd4f",
-    "text": "GCE-only flag `cloud-provider-gce-lb-src-cidrs` becomes optional for external cloud providers.",
-    "markdown": "GCE-only flag `cloud-provider-gce-lb-src-cidrs` becomes optional for external cloud providers. ([#76627](https://github.com/kubernetes/kubernetes/pull/76627), [@timoreimann](https://github.com/timoreimann))",
-    "author": "timoreimann",
-    "author_url": "https://github.com/timoreimann",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76627",
-    "pr_number": 76627,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "76628": {
-    "commit": "02ffdd1d5bbfdd7d6e129caaf3a5d71550817a61",
-    "text": "Juju provider source moved to the Charmed Kubernetes org",
-    "markdown": "Juju provider source moved to the Charmed Kubernetes org ([#76628](https://github.com/kubernetes/kubernetes/pull/76628), [@kwmonroe](https://github.com/kwmonroe))",
-    "author": "kwmonroe",
-    "author_url": "https://github.com/kwmonroe",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76628",
-    "pr_number": 76628,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.2"
-  },
-  "76636": {
-    "commit": "9746a40e44a3b208b67e30148e52f3dc6f7cfe20",
-    "text": "Kubeadm will now include the missing certificate key if it is unable to find an expected key during `kubeadm join` when used with the `--experimental-control-plane` flow",
-    "markdown": "Kubeadm will now include the missing certificate key if it is unable to find an expected key during `kubeadm join` when used with the `--experimental-control-plane` flow ([#76636](https://github.com/kubernetes/kubernetes/pull/76636), [@mdaniel](https://github.com/mdaniel))",
-    "author": "mdaniel",
-    "author_url": "https://github.com/mdaniel",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76636",
-    "pr_number": 76636,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.2"
-  },
-  "76640": {
-    "commit": "a961c13db53b4c28770b31331b5e581864dafcd2",
-    "text": "k8s-dns-node-cache image version v1.15.1",
-    "markdown": "k8s-dns-node-cache image version v1.15.1 ([#76640](https://github.com/kubernetes/kubernetes/pull/76640), [@george-angel](https://github.com/george-angel))",
-    "author": "george-angel",
-    "author_url": "https://github.com/george-angel",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76640",
-    "pr_number": 76640,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.2"
-  },
-  "76646": {
-    "commit": "e6188f8c295202f9dc35f30b65a4ac6d2180c3df",
-    "text": "Support print volumeMode using `kubectl get pv/pvc -o wide`",
-    "markdown": "Support print volumeMode using `kubectl get pv/pvc -o wide` ([#76646](https://github.com/kubernetes/kubernetes/pull/76646), [@cwdsuzhou](https://github.com/cwdsuzhou))\n\n  Courtesy of SIG CLI",
-    "author": "cwdsuzhou",
-    "author_url": "https://github.com/cwdsuzhou",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76646",
-    "pr_number": 76646,
-    "kinds": ["feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "76656": {
-    "commit": "f0a4672e6db7196312ef85731c19eadcbee1da9d",
-    "text": "Switch to instance-level update APIs for Azure VMSS loadbalancer operations",
-    "markdown": "Switch to instance-level update APIs for Azure VMSS loadbalancer operations ([#76656](https://github.com/kubernetes/kubernetes/pull/76656), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76656",
-    "pr_number": 76656,
-    "areas": ["cloudprovider", "dependency"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "76659": {
-    "commit": "21bec91e895a828ee8ed8d45a7d58279b8cae46f",
-    "text": "Signal handling is initialized within hyperkube commands that require it (apiserver, kubelet)",
-    "markdown": "Signal handling is initialized within hyperkube commands that require it (apiserver, kubelet) ([#76659](https://github.com/kubernetes/kubernetes/pull/76659), [@S-Chan](https://github.com/S-Chan))",
-    "author": "S-Chan",
-    "author_url": "https://github.com/S-Chan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76659",
-    "pr_number": 76659,
-    "areas": ["apiserver", "kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "node", "release"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "76675": {
-    "commit": "5ccda18fae27c474ec4e84537a8fe6cfa9d5dd96",
-    "text": "Fixes an error with stuck informers when an etcd watch receives update or delete events with missing data",
-    "markdown": "Fixes an error with stuck informers when an etcd watch receives update or delete events with missing data ([#76675](https://github.com/kubernetes/kubernetes/pull/76675), [@ryanmcnamara](https://github.com/ryanmcnamara))",
-    "author": "ryanmcnamara",
-    "author_url": "https://github.com/ryanmcnamara",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76675",
-    "pr_number": 76675,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.2"
-  },
-  "76691": {
-    "commit": "8f09fd6b2ab2bd52c000e8de06cb90f5c2bdc727",
-    "text": "Ensure the backend pools are set correctly for Azure SLB with multiple backend pools (e.g. outbound rules)",
-    "markdown": "Ensure the backend pools are set correctly for Azure SLB with multiple backend pools (e.g. outbound rules) ([#76691](https://github.com/kubernetes/kubernetes/pull/76691), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76691",
-    "pr_number": 76691,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.2"
-  },
-  "76710": {
-    "commit": "332d62a94a2bc8a7d0c69ca85fcd5c458816d677",
-    "text": "Introduce the v1beta2 config format to kubeadm.",
-    "markdown": "Introduce the v1beta2 config format to kubeadm. ([#76710](https://github.com/kubernetes/kubernetes/pull/76710), [@rosti](https://github.com/rosti))",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76710",
-    "pr_number": 76710,
-    "areas": ["kubeadm"],
-    "kinds": ["api-change"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.2"
-  },
-  "76711": {
-    "commit": "c58864de3f5f990b7dbcd25fcc93214eee03d127",
-    "text": "kube-up.sh no longer supports \"centos\" and \"local\" providers",
-    "markdown": "kube-up.sh no longer supports \"centos\" and \"local\" providers ([#76711](https://github.com/kubernetes/kubernetes/pull/76711), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76711",
-    "pr_number": 76711,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "76722": {
-    "commit": "257e6863b8c33a24453fac540df068d1a6657cd4",
-    "text": "Windows nodes on GCE use a known-working 1809 image rather than the latest 1809 image.",
-    "markdown": "Windows nodes on GCE use a known-working 1809 image rather than the latest 1809 image. ([#76722](https://github.com/kubernetes/kubernetes/pull/76722), [@pjh](https://github.com/pjh))",
-    "author": "pjh",
-    "author_url": "https://github.com/pjh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76722",
-    "pr_number": 76722,
-    "kinds": ["bug", "failing-test"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "76732": {
-    "commit": "a0b8d1cbb883483a8fc77fc4a1cc830fff290b32",
-    "text": "kube-proxy: os exit when CleanupAndExit is set to true",
-    "markdown": "kube-proxy: os exit when CleanupAndExit is set to true ([#76732](https://github.com/kubernetes/kubernetes/pull/76732), [@JieJhih](https://github.com/JieJhih))",
-    "author": "JieJhih",
-    "author_url": "https://github.com/JieJhih",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76732",
-    "pr_number": 76732,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.2"
-  },
-  "76749": {
-    "commit": "f5d958af4bbb38c562f4f11df6103790685abcc8",
-    "text": "Limit use of tags when calling EC2 API to prevent API throttling for very large clusters",
-    "markdown": "Limit use of tags when calling EC2 API to prevent API throttling for very large clusters ([#76749](https://github.com/kubernetes/kubernetes/pull/76749), [@mcrute](https://github.com/mcrute))",
-    "author": "mcrute",
-    "author_url": "https://github.com/mcrute",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76749",
-    "pr_number": 76749,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.2"
-  },
-  "76762": {
-    "commit": "d2ce69d9ad3d28c554c9a9560c15947181029ca8",
-    "text": "[fluentd-gcp addon] Bump fluentd-gcp-scaler to v0.5.2 to pick up security fixes.",
-    "markdown": "[fluentd-gcp addon] Bump fluentd-gcp-scaler to v0.5.2 to pick up security fixes. ([#76762](https://github.com/kubernetes/kubernetes/pull/76762), [@serathius](https://github.com/serathius))",
-    "author": "serathius",
-    "author_url": "https://github.com/serathius",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76762",
-    "pr_number": 76762,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.2"
-  },
-  "76788": {
-    "commit": "b95fca093d29a0c63558138e1bfa3b434168e994",
-    "text": "Clean links handling in cp's tar code",
-    "markdown": "Clean links handling in cp's tar code ([#76788](https://github.com/kubernetes/kubernetes/pull/76788), [@soltysh](https://github.com/soltysh))",
-    "author": "soltysh",
-    "author_url": "https://github.com/soltysh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76788",
-    "pr_number": 76788,
-    "areas": ["kubectl", "security"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.2"
-  },
-  "76819": {
-    "commit": "b32b742d978ebdb195936ddfe28fd6efcf2a454b",
-    "text": "[metrics-server addon] Restore connecting to nodes via IP addresses",
-    "markdown": "[metrics-server addon] Restore connecting to nodes via IP addresses ([#76819](https://github.com/kubernetes/kubernetes/pull/76819), [@serathius](https://github.com/serathius))",
-    "author": "serathius",
-    "author_url": "https://github.com/serathius",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76819",
-    "pr_number": 76819,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "76821": {
-    "commit": "36ccff1b27b6aee5c5599334842cb2b8cf8063f3",
-    "text": "kubeadm: Improve resiliency when it comes to updating the `kubeadm-config` config map upon new control plane joins or resets. This allows for safe multiple control plane joins and/or resets.",
-    "markdown": "kubeadm: Improve resiliency when it comes to updating the `kubeadm-config` config map upon new control plane joins or resets. This allows for safe multiple control plane joins and/or resets. ([#76821](https://github.com/kubernetes/kubernetes/pull/76821), [@ereslibre](https://github.com/ereslibre))",
-    "author": "ereslibre",
-    "author_url": "https://github.com/ereslibre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76821",
-    "pr_number": 76821,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.2"
-  },
-  "76848": {
-    "commit": "ccf9556ddcfdad779a5e072bb8415bbe9f559381",
-    "text": "Fixed a kubemark panic when hollow-node is morphed as proxy.",
-    "markdown": "Fixed a kubemark panic when hollow-node is morphed as proxy. ([#76848](https://github.com/kubernetes/kubernetes/pull/76848), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76848",
-    "pr_number": 76848,
-    "kinds": ["bug"],
-    "sigs": ["scalability"],
-    "release_version": "1.14.2"
-  },
-  "76849": {
-    "commit": "716344fd7d68f92d9a16280be1feae0ffe122bb4",
-    "text": "Admission webhooks are now properly called for `scale` and `deployments/rollback` subresources",
-    "markdown": "Admission webhooks are now properly called for `scale` and `deployments/rollback` subresources ([#76849](https://github.com/kubernetes/kubernetes/pull/76849), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery, SIG Auth, and SIG Testing",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76849",
-    "pr_number": 76849,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "auth", "testing"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "76850": {
-    "commit": "aa11d0edef0d73afaf1c54f8ec37c6a11a1e4891",
-    "text": "GCE/Windows: add support for stackdriver logging agent",
-    "markdown": "GCE/Windows: add support for stackdriver logging agent ([#76850](https://github.com/kubernetes/kubernetes/pull/76850), [@yujuhong](https://github.com/yujuhong))",
-    "author": "yujuhong",
-    "author_url": "https://github.com/yujuhong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76850",
-    "pr_number": 76850,
-    "kinds": ["bug"],
-    "release_version": "1.14.2"
-  },
-  "76862": {
-    "commit": "c88b7cdd58cb0efc79a9c1009eb5f927c3fc2b7e",
-    "text": "kubeadm: kubeadm upgrade now renews all the certificates used by one component before upgrading the component itself, with the exception of certificates signed by external CAs. User can eventually opt-out from certificate renewal during upgrades by setting the new flag --certificate-renewal to false.",
-    "markdown": "kubeadm: kubeadm upgrade now renews all the certificates used by one component before upgrading the component itself, with the exception of certificates signed by external CAs. User can eventually opt-out from certificate renewal during upgrades by setting the new flag --certificate-renewal to false. ([#76862](https://github.com/kubernetes/kubernetes/pull/76862), [@fabriziopandini](https://github.com/fabriziopandini))",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76862",
-    "pr_number": 76862,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.2"
-  },
-  "76865": {
-    "commit": "c262f9c7d9c4d6468f4dc28b2e61976dd6ff6115",
-    "text": "kubeadm: kubeadm alpha cert renew now ignores certificates signed by external CAs",
-    "markdown": "kubeadm: kubeadm alpha cert renew now ignores certificates signed by external CAs ([#76865](https://github.com/kubernetes/kubernetes/pull/76865), [@fabriziopandini](https://github.com/fabriziopandini))",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76865",
-    "pr_number": 76865,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.2"
-  },
-  "76909": {
-    "commit": "9b437f95207c04bf2f25ef3110fac9b356d1fa91",
-    "text": "Kubemark scripts have been fixed for IKS clusters.",
-    "markdown": "Kubemark scripts have been fixed for IKS clusters. ([#76909](https://github.com/kubernetes/kubernetes/pull/76909), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76909",
-    "pr_number": 76909,
-    "areas": ["provider/ibmcloud", "test"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "scalability", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "76910": {
-    "commit": "90fbbee12950f336db2da94dda7beb87846f94e0",
-    "text": "Validating admission webhooks are now properly called for CREATE operations on the following resources: pods/binding, pods/eviction, bindings",
-    "markdown": "Validating admission webhooks are now properly called for CREATE operations on the following resources: pods/binding, pods/eviction, bindings ([#76910](https://github.com/kubernetes/kubernetes/pull/76910), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76910",
-    "pr_number": 76910,
-    "areas": ["admission-control", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["testing"],
-    "release_version": "1.14.2"
-  },
-  "76914": {
-    "commit": "219b166cb172344ee666a107161c07ba3805bd67",
-    "text": "Fix OpenID Connect (OIDC) token refresh when the client secret contains a special character.",
-    "markdown": "Fix OpenID Connect (OIDC) token refresh when the client secret contains a special character. ([#76914](https://github.com/kubernetes/kubernetes/pull/76914), [@tsuna](https://github.com/tsuna))",
-    "author": "tsuna",
-    "author_url": "https://github.com/tsuna",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76914",
-    "pr_number": 76914,
-    "areas": ["apiserver", "cloudprovider", "dependency"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth", "cli", "cloud-provider", "cluster-lifecycle", "node"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "76920": {
-    "commit": "35b278447cd811f576d6491348ff321b7b9b9089",
-    "text": "kubeadm: support sub-domain wildcards in certificate SANs",
-    "markdown": "kubeadm: support sub-domain wildcards in certificate SANs ([#76920](https://github.com/kubernetes/kubernetes/pull/76920), [@sempr](https://github.com/sempr))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "sempr",
-    "author_url": "https://github.com/sempr",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76920",
-    "pr_number": 76920,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "76953": {
-    "commit": "0ff81d04dc5ff7870f8b0ced66a67b8c68eb7065",
-    "text": "Remove hyperkube short aliases from source code, Because hyperkube docker image currently create these aliases. If you are currently using /hyperkube {apiserver,scheduler,controller-manager,proxy} please switch to using the correct binary names.",
-    "markdown": "Remove hyperkube short aliases from source code, Because hyperkube docker image currently create these aliases. If you are currently using /hyperkube {apiserver,scheduler,controller-manager,proxy} please switch to using the correct binary names. ([#76953](https://github.com/kubernetes/kubernetes/pull/76953), [@Rand01ph](https://github.com/Rand01ph))\n\n  Courtesy of SIG Release",
-    "author": "Rand01ph",
-    "author_url": "https://github.com/Rand01ph",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76953",
-    "pr_number": 76953,
-    "kinds": ["cleanup"],
-    "sigs": ["release"],
-    "action_required": true,
-    "release_version": "1.14.2"
-  },
-  "76959": {
-    "commit": "279d4e1ca3b550ef7d193e93854e6eb10b937661",
-    "text": "Validating admission webhooks are now properly called for CREATE operations on the following resources: tokenreviews, subjectaccessreviews, localsubjectaccessreviews, selfsubjectaccessreviews, selfsubjectrulesreviews",
-    "markdown": "Validating admission webhooks are now properly called for CREATE operations on the following resources: tokenreviews, subjectaccessreviews, localsubjectaccessreviews, selfsubjectaccessreviews, selfsubjectrulesreviews ([#76959](https://github.com/kubernetes/kubernetes/pull/76959), [@sbezverk](https://github.com/sbezverk))",
-    "author": "sbezverk",
-    "author_url": "https://github.com/sbezverk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76959",
-    "pr_number": 76959,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["auth", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "76969": {
-    "commit": "28172ec17c088527bf543e65d2a2019ba7add20a",
-    "text": "Fixes a bug where dry-run is not honored for pod/eviction sub-resource.",
-    "markdown": "Fixes a bug where dry-run is not honored for pod/eviction sub-resource. ([#76969](https://github.com/kubernetes/kubernetes/pull/76969), [@apelisse](https://github.com/apelisse))",
-    "author": "apelisse",
-    "author_url": "https://github.com/apelisse",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76969",
-    "pr_number": 76969,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.2"
-  },
-  "76976": {
-    "commit": "805b6cbf83c8a60db735dd94e222432d451f634a",
-    "text": "`aws-cloud-provider` service account in the `kube-system` namespace need to be granted with list node permission with this optimization",
-    "markdown": "`aws-cloud-provider` service account in the `kube-system` namespace need to be granted with list node permission with this optimization ([#76976](https://github.com/kubernetes/kubernetes/pull/76976), [@zhan849](https://github.com/zhan849))",
-    "author": "zhan849",
-    "author_url": "https://github.com/zhan849",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76976",
-    "pr_number": 76976,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.2"
-  },
-  "76988": {
-    "commit": "2e96bf597eaa4e74650833916d6d984e93b77654",
-    "text": "---\nkind: StorageClass\napiVersion: storage.k8s.io/v1\nmetadata:\n  name: azurefile\nprovisioner: kubernetes.io/azure-file\nparameters:\n  skuName: Standard_LRS\n  resourceGroup: EXISTING_RESOURCE_GROUP_NAME\n  storageAccount: EXISTING_STORAGE_ACCOUNT_NAME\n  shareName: SHARE_NAME",
-    "markdown": "---\n  kind: StorageClass\n  apiVersion: storage.k8s.io/v1\n  metadata:\n    name: azurefile\n  provisioner: kubernetes.io/azure-file\n  parameters:\n    skuName: Standard_LRS\n    resourceGroup: EXISTING_RESOURCE_GROUP_NAME\n    storageAccount: EXISTING_STORAGE_ACCOUNT_NAME\n    shareName: SHARE_NAME ([#76988](https://github.com/kubernetes/kubernetes/pull/76988), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76988",
-    "pr_number": 76988,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "77010": {
-    "commit": "36e06bbb1fd3fc430ff4f8990684bbd53cf80e79",
-    "text": "Report cp errors consistently",
-    "markdown": "Report cp errors consistently ([#77010](https://github.com/kubernetes/kubernetes/pull/77010), [@soltysh](https://github.com/soltysh))",
-    "author": "soltysh",
-    "author_url": "https://github.com/soltysh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77010",
-    "pr_number": 77010,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.2"
-  },
-  "77022": {
-    "commit": "aefa6d4492dd014c27cfc6f08ee5fe94384f87a3",
-    "text": "API requests rejected by admission webhooks which specify an http status code \u003c 400 are now assigned a 400 status code.",
-    "markdown": "API requests rejected by admission webhooks which specify an http status code \u003c 400 are now assigned a 400 status code. ([#77022](https://github.com/kubernetes/kubernetes/pull/77022), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77022",
-    "pr_number": 77022,
-    "areas": ["admission-control", "apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.2"
-  },
-  "77051": {
-    "commit": "59a86822cb966ffb77a702321c89bb4e6f1c1c89",
-    "text": "kubeadm: bump the minimum supported Docker version to 1.13.1",
-    "markdown": "kubeadm: bump the minimum supported Docker version to 1.13.1 ([#77051](https://github.com/kubernetes/kubernetes/pull/77051), [@chenzhiwei](https://github.com/chenzhiwei))",
-    "author": "chenzhiwei",
-    "author_url": "https://github.com/chenzhiwei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77051",
-    "pr_number": 77051,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.2"
-  },
-  "77070": {
-    "commit": "9e29c3e39f916fe67654c9e06ada23e42217e532",
-    "text": "Upgrade go-autorest to v11.1.2",
-    "markdown": "Upgrade go-autorest to v11.1.2 ([#77070](https://github.com/kubernetes/kubernetes/pull/77070), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77070",
-    "pr_number": 77070,
-    "areas": ["apiserver", "cloudprovider", "dependency"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "77099": {
-    "commit": "2776cc1e726fb9fa74d3deddcd302b17a4cfb278",
-    "text": "Rancher credential provider has now been removed",
-    "markdown": "Rancher credential provider has now been removed ([#77099](https://github.com/kubernetes/kubernetes/pull/77099), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77099",
-    "pr_number": 77099,
-    "areas": ["dependency", "kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["cloud-provider", "node"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "77136": {
-    "commit": "ba28e0e0fde4072aecb94e7d596e97d1c976a49e",
-    "text": "Promote meta.k8s.io/v1beta1 Table and PartialObjectMetadata to v1.",
-    "markdown": "Promote meta.k8s.io/v1beta1 Table and PartialObjectMetadata to v1. ([#77136](https://github.com/kubernetes/kubernetes/pull/77136), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77136",
-    "pr_number": 77136,
-    "areas": ["dependency"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.2"
-  },
-  "77139": {
-    "commit": "83f769c2a3111086af99a079ef276aefeb0366de",
-    "text": "ingress objects are now persisted in etcd using the networking.k8s.io/v1beta1 version",
-    "markdown": "ingress objects are now persisted in etcd using the networking.k8s.io/v1beta1 version ([#77139](https://github.com/kubernetes/kubernetes/pull/77139), [@cmluciano](https://github.com/cmluciano))",
-    "author": "cmluciano",
-    "author_url": "https://github.com/cmluciano",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77139",
-    "pr_number": 77139,
-    "areas": ["apiserver", "test"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "network", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "77144": {
-    "commit": "167bc954fdb1a1d592b343de7a05fa57f58a5458",
-    "text": "When specifying an invalid value for a label, it was not always\nclear which label the value was specified for. Starting with this release, the\nlabel's key is included in such error messages, which makes debugging easier.",
-    "markdown": "When specifying an invalid value for a label, it was not always\n  clear which label the value was specified for. Starting with this release, the\n  label's key is included in such error messages, which makes debugging easier. ([#77144](https://github.com/kubernetes/kubernetes/pull/77144), [@kenegozi](https://github.com/kenegozi))",
-    "author": "kenegozi",
-    "author_url": "https://github.com/kenegozi",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77144",
-    "pr_number": 77144,
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.2"
-  },
-  "77180": {
-    "commit": "f9c9ecdbb62e5510dbdadf7ef4d3d48956562af7",
-    "text": "kubeadm: kubeadm alpha certs renew and kubeadm upgrade now supports renews of certificates embedded in KubeConfig files managed by kubeadm; this does not apply to certificates signed by external CAs.",
-    "markdown": "kubeadm: kubeadm alpha certs renew and kubeadm upgrade now supports renews of certificates embedded in KubeConfig files managed by kubeadm; this does not apply to certificates signed by external CAs. ([#77180](https://github.com/kubernetes/kubernetes/pull/77180), [@fabriziopandini](https://github.com/fabriziopandini))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77180",
-    "pr_number": 77180,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "77187": {
-    "commit": "16193d83a4725242768b222c1f3f8346c8f9b6ae",
-    "text": "fix azure disk list corruption issue",
-    "markdown": "fix azure disk list corruption issue ([#77187](https://github.com/kubernetes/kubernetes/pull/77187), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77187",
-    "pr_number": 77187,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.2"
-  },
-  "77207": {
-    "commit": "8efea56ac8e07c57ba93d5e6764e09b46d19f87e",
-    "text": "- Add a condition NonStructuralSchema to CustomResourceDefinition listing Structural Schema violations as defined in KEP https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190425-structural-openapi.md. CRD authors should update their validation schemas to be structural in order to participate in future CRD features.\n- CRD validation schemas get support for `x-kubernetes-embedded-resource: true` to specify `RawExtensions` in OpenAPI and to enforce coercing and validation of TypeMeta and ObjectMeta.\n- CRD validation schemas get support for `x-kubernetes-int-or-string: true` to specify `intstr.IntOrString` in OpenAPI.\n- CRD validation schemas get support for `x-kubernetes-preserve-unknown-fields: true` to control pruning via OpenAPI for parts of CustomResources.",
-    "markdown": "- Add a condition NonStructuralSchema to CustomResourceDefinition listing Structural Schema violations as defined in KEP https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190425-structural-openapi.md. CRD authors should update their validation schemas to be structural in order to participate in future CRD features.\n  - CRD validation schemas get support for `x-kubernetes-embedded-resource: true` to specify `RawExtensions` in OpenAPI and to enforce coercing and validation of TypeMeta and ObjectMeta.\n  - CRD validation schemas get support for `x-kubernetes-int-or-string: true` to specify `intstr.IntOrString` in OpenAPI.\n  - CRD validation schemas get support for `x-kubernetes-preserve-unknown-fields: true` to control pruning via OpenAPI for parts of CustomResources. ([#77207](https://github.com/kubernetes/kubernetes/pull/77207), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77207",
-    "pr_number": 77207,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.2"
-  },
-  "77208": {
-    "commit": "e871241268c4aed15cd7d55692289297513bec87",
-    "text": "enhance the daemonset sync logic in clock-skew scenario",
-    "markdown": "enhance the daemonset sync logic in clock-skew scenario ([#77208](https://github.com/kubernetes/kubernetes/pull/77208), [@DaiHao](https://github.com/DaiHao))",
-    "author": "DaiHao",
-    "author_url": "https://github.com/DaiHao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77208",
-    "pr_number": 77208,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.14.2"
-  },
-  "77210": {
-    "commit": "5f6d9b614ed9637cadb8a6497293454f2142b348",
-    "text": "Add etag for NSG updates so as to fix nsg race condition",
-    "markdown": "Add etag for NSG updates so as to fix nsg race condition ([#77210](https://github.com/kubernetes/kubernetes/pull/77210), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77210",
-    "pr_number": 77210,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.2"
-  },
-  "77224": {
-    "commit": "62020ebe38ac5bbe72ca614974c6ebbe22a61986",
-    "text": "Fixes segmentation fault issue with Protobuf library when log entries are deeply nested.",
-    "markdown": "Fixes segmentation fault issue with Protobuf library when log entries are deeply nested. ([#77224](https://github.com/kubernetes/kubernetes/pull/77224), [@qingling128](https://github.com/qingling128))",
-    "author": "qingling128",
-    "author_url": "https://github.com/qingling128",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77224",
-    "pr_number": 77224,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.2"
-  },
-  "77245": {
-    "commit": "c4196408913d3ee2525d705d1636879b47b0c7cf",
-    "text": "fix issue that pull image failed from a cross-subscription Azure Container Registry when using MSI to authenticate",
-    "markdown": "fix issue that pull image failed from a cross-subscription Azure Container Registry when using MSI to authenticate ([#77245](https://github.com/kubernetes/kubernetes/pull/77245), [@norshtein](https://github.com/norshtein))",
-    "author": "norshtein",
-    "author_url": "https://github.com/norshtein",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77245",
-    "pr_number": 77245,
-    "kinds": ["bug"],
-    "release_version": "1.14.2"
-  },
-  "77267": {
-    "commit": "1e015237c702902133e655a8712b9004175745d0",
-    "text": "Preserves existing namespace information in manifests when running `kubectl set ... --local` commands",
-    "markdown": "Preserves existing namespace information in manifests when running `kubectl set ... --local` commands ([#77267](https://github.com/kubernetes/kubernetes/pull/77267), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77267",
-    "pr_number": 77267,
-    "areas": ["kubectl", "test"],
-    "kinds": ["bug"],
-    "sigs": ["cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "77278": {
-    "commit": "005eb53ea1d9a0fac4f008f79a9ee559436def53",
-    "text": "API paging is now enabled by default in k8s.io/apiserver recommended options, and in k8s.io/sample-apiserver",
-    "markdown": "API paging is now enabled by default in k8s.io/apiserver recommended options, and in k8s.io/sample-apiserver ([#77278](https://github.com/kubernetes/kubernetes/pull/77278), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77278",
-    "pr_number": 77278,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.2"
-  },
-  "77282": {
-    "commit": "6027a38e78b4c257028d1a48fe3d90d8c02d99c4",
-    "text": "Bump addon-manager to v9.0.1\n- Rebase image on debian-base:v1.0.0",
-    "markdown": "Bump addon-manager to v9.0.1\n  - Rebase image on debian-base:v1.0.0 ([#77282](https://github.com/kubernetes/kubernetes/pull/77282), [@MrHohn](https://github.com/MrHohn))",
-    "author": "MrHohn",
-    "author_url": "https://github.com/MrHohn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77282",
-    "pr_number": 77282,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "77303": {
-    "commit": "53b41dd1fe1444b8c9bca948f5e469058474ff91",
-    "text": "Fixes spurious error messages about failing to clean up iptables rules when using iptables 1.8.",
-    "markdown": "Fixes spurious error messages about failing to clean up iptables rules when using iptables 1.8. ([#77303](https://github.com/kubernetes/kubernetes/pull/77303), [@danwinship](https://github.com/danwinship))",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77303",
-    "pr_number": 77303,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.2"
-  },
-  "77317": {
-    "commit": "9e52832c52c84c4b4b3dee6733ad392ff17e1753",
-    "text": "Fix bug with block volume expansion",
-    "markdown": "Fix bug with block volume expansion ([#77317](https://github.com/kubernetes/kubernetes/pull/77317), [@gnufied](https://github.com/gnufied))",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77317",
-    "pr_number": 77317,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "77335": {
-    "commit": "ce9ea5ea4154829fe00efe016b0860d7adca9074",
-    "text": "This PR removes unused soak test cauldron",
-    "markdown": "This PR removes unused soak test cauldron ([#77335](https://github.com/kubernetes/kubernetes/pull/77335), [@loqutus](https://github.com/loqutus))",
-    "author": "loqutus",
-    "author_url": "https://github.com/loqutus",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77335",
-    "pr_number": 77335,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["testing"],
-    "release_version": "1.14.2"
-  },
-  "77347": {
-    "commit": "df29573ec50375e831d833e27c63c23a88b983e8",
-    "text": "Fix kubectl describe CronJobs error of `Successful Job History Limit`.",
-    "markdown": "Fix kubectl describe CronJobs error of `Successful Job History Limit`. ([#77347](https://github.com/kubernetes/kubernetes/pull/77347), [@danielqsj](https://github.com/danielqsj))",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77347",
-    "pr_number": 77347,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.2"
-  },
-  "77368": {
-    "commit": "5bbad4fd4637e7bdaa3666e636539d9e2b9e031f",
-    "text": "Fixes incorrect handling by kubectl of custom resources whose Kind is \"Status\"",
-    "markdown": "Fixes incorrect handling by kubectl of custom resources whose Kind is \"Status\" ([#77368](https://github.com/kubernetes/kubernetes/pull/77368), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77368",
-    "pr_number": 77368,
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.2"
-  },
-  "77378": {
-    "commit": "6a23a6278f48ef286b860159a9cab81d71e30640",
-    "text": "GCE/Windows: force kill Stackdriver logging processes when the service cannot be stopped",
-    "markdown": "GCE/Windows: force kill Stackdriver logging processes when the service cannot be stopped ([#77378](https://github.com/kubernetes/kubernetes/pull/77378), [@yujuhong](https://github.com/yujuhong))",
-    "author": "yujuhong",
-    "author_url": "https://github.com/yujuhong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77378",
-    "pr_number": 77378,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.2"
-  },
-  "77412": {
-    "commit": "1dd550cf7c1b9ab54c931d4e0976197f31d1b304",
-    "text": "The GCERegionalPersistentDisk feature gate (GA in 1.13) can no longer be disabled. The feature gate will be removed in v1.17.",
-    "markdown": "The GCERegionalPersistentDisk feature gate (GA in 1.13) can no longer be disabled. The feature gate will be removed in v1.17. ([#77412](https://github.com/kubernetes/kubernetes/pull/77412), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77412",
-    "pr_number": 77412,
-    "areas": ["cloudprovider"],
-    "kinds": ["cleanup"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "77415": {
-    "commit": "291f2f863ab6f73787962f65b0f38998f3c9ac00",
-    "text": "Allow to define kubeconfig file for OpenStack cloud provider.",
-    "markdown": "Allow to define kubeconfig file for OpenStack cloud provider. ([#77415](https://github.com/kubernetes/kubernetes/pull/77415), [@Fedosin](https://github.com/Fedosin))\n\n  Courtesy of SIG Cloud Provider, and SIG Storage",
-    "author": "Fedosin",
-    "author_url": "https://github.com/Fedosin",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77415",
-    "pr_number": 77415,
-    "areas": ["cloudprovider"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider", "storage"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "77426": {
-    "commit": "946087b422e38dba37a8418860b0f2ede8597394",
-    "text": "If a pod has a running instance, the stats of its previously terminated instances will not show up in the kubelet summary stats any more for CRI runtimes like containerd and cri-o.\n\nThis keeps the behavior consistent with Docker integration, and fixes an issue that some container Prometheus metrics don't work when there are summary stats for multiple instances of the same pod.",
-    "markdown": "If a pod has a running instance, the stats of its previously terminated instances will not show up in the kubelet summary stats any more for CRI runtimes like containerd and cri-o.\n  \n  This keeps the behavior consistent with Docker integration, and fixes an issue that some container Prometheus metrics don't work when there are summary stats for multiple instances of the same pod. ([#77426](https://github.com/kubernetes/kubernetes/pull/77426), [@Random-Liu](https://github.com/Random-Liu))",
-    "author": "Random-Liu",
-    "author_url": "https://github.com/Random-Liu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77426",
-    "pr_number": 77426,
-    "areas": ["kubelet"],
-    "kinds": ["bug", "failing-test"],
-    "sigs": ["instrumentation", "node"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "77447": {
-    "commit": "0befec7fb6e53335e6672c464ddb3fb4343bfef2",
-    "text": "API servers using the default Google Compute Engine bootstrapping scripts will have their insecure port (`:8080`) disabled by default. To enable the insecure port, set `ENABLE_APISERVER_INSECURE_PORT=true` in kube-env or as an environment variable.",
-    "markdown": "API servers using the default Google Compute Engine bootstrapping scripts will have their insecure port (`:8080`) disabled by default. To enable the insecure port, set `ENABLE_APISERVER_INSECURE_PORT=true` in kube-env or as an environment variable. ([#77447](https://github.com/kubernetes/kubernetes/pull/77447), [@dekkagaijin](https://github.com/dekkagaijin))",
-    "author": "dekkagaijin",
-    "author_url": "https://github.com/dekkagaijin",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77447",
-    "pr_number": 77447,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.2"
-  },
-  "77448": {
-    "commit": "274876ef9af526575f480c977f7dd7c4bb26e760",
-    "text": "Clients may now request that API objects are converted to the `v1.Table` and `v1.PartialObjectMetadata` forms for generic access to objects.",
-    "markdown": "Clients may now request that API objects are converted to the `v1.Table` and `v1.PartialObjectMetadata` forms for generic access to objects. ([#77448](https://github.com/kubernetes/kubernetes/pull/77448), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77448",
-    "pr_number": 77448,
-    "areas": ["apiserver", "kubectl", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "77457": {
-    "commit": "1ce103a41f98af6b7ec7f574fbbf7ef7356c5c82",
-    "text": "Implement un-reserve extension point for the scheduling framework.",
-    "markdown": "Implement un-reserve extension point for the scheduling framework. ([#77457](https://github.com/kubernetes/kubernetes/pull/77457), [@danielqsj](https://github.com/danielqsj))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77457",
-    "pr_number": 77457,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "77458": {
-    "commit": "b34d7ac0ce0c43ee4bec1e15185b13e44a2f7843",
-    "text": "GCE clusters will include some IP ranges that are not in used on the public Internet to the list of non-masq IPs.\nBump ip-masq-agent version to v2.3.0 with flag `nomasq-all-reserved-ranges` turned on.",
-    "markdown": "GCE clusters will include some IP ranges that are not in used on the public Internet to the list of non-masq IPs.\n  Bump ip-masq-agent version to v2.3.0 with flag `nomasq-all-reserved-ranges` turned on. ([#77458](https://github.com/kubernetes/kubernetes/pull/77458), [@grayluck](https://github.com/grayluck))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Network",
-    "author": "grayluck",
-    "author_url": "https://github.com/grayluck",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77458",
-    "pr_number": 77458,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "77483": {
-    "commit": "ed239cefa6b3add3d53d55e02fa9c28a0bbfc25a",
-    "text": "remove VM API call dep in azure disk WaitForAttach",
-    "markdown": "remove VM API call dep in azure disk WaitForAttach ([#77483](https://github.com/kubernetes/kubernetes/pull/77483), [@andyzhangx](https://github.com/andyzhangx))\n\n  Courtesy of SIG Storage",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77483",
-    "pr_number": 77483,
-    "kinds": ["bug", "feature"],
-    "sigs": ["storage"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "77490": {
-    "commit": "b9bde60cfd01cf657fec0dadba826c57aec4dcf9",
-    "text": "Fix race conditions for Azure loadbalancer and route updates.",
-    "markdown": "Fix race conditions for Azure loadbalancer and route updates. ([#77490](https://github.com/kubernetes/kubernetes/pull/77490), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77490",
-    "pr_number": 77490,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.2"
-  },
-  "77491": {
-    "commit": "1c689c2f49b5e03d4b12c5a4247ed30ac1c1bddb",
-    "text": "add operation name for vm/vmss update operations in prometheus metrics",
-    "markdown": "add operation name for vm/vmss update operations in prometheus metrics ([#77491](https://github.com/kubernetes/kubernetes/pull/77491), [@andyzhangx](https://github.com/andyzhangx))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77491",
-    "pr_number": 77491,
-    "areas": ["cloudprovider"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "77495": {
-    "commit": "8ea9edbb0290e9de1e6d274e816a4002892cca6f",
-    "text": "Client:\n Version:           19.03.0-beta3\n API version:       1.40\n Go version:        go1.12.4\n Git commit:        c55e026\n Built:             Thu Apr 25 03:03:18 2019\n OS/Arch:           linux/amd64\n Experimental:      false\n\nServer:\n Engine:\n  Version:          19.03.0-beta3\n  API version:      1.40 (minimum version 1.12)\n  Go version:       go1.12.4\n  Git commit:       c55e026\n  Built:            Thu Apr 25 03:01:56 2019\n  OS/Arch:          linux/amd64\n  Experimental:     false\n containerd:\n  Version:          1.2.5\n  GitCommit:        bb71b10fd8f58240ca47fbb579b9d1028eea7c84\n runc:\n  Version:          1.0.0-rc6+dev\n  GitCommit:        2b18fe1d885ee5083ef9f0838fee39b62d653e30\n docker-init:\n  Version:          0.18.0\n  GitCommit:        fec3683",
-    "markdown": "Client:\n   Version:           19.03.0-beta3\n   API version:       1.40\n   Go version:        go1.12.4\n   Git commit:        c55e026\n   Built:             Thu Apr 25 03:03:18 2019\n   OS/Arch:           linux/amd64\n   Experimental:      false\n  \n  Server:\n   Engine:\n    Version:          19.03.0-beta3\n    API version:      1.40 (minimum version 1.12)\n    Go version:       go1.12.4\n    Git commit:       c55e026\n    Built:            Thu Apr 25 03:01:56 2019\n    OS/Arch:          linux/amd64\n    Experimental:     false\n   containerd:\n    Version:          1.2.5\n    GitCommit:        bb71b10fd8f58240ca47fbb579b9d1028eea7c84\n   runc:\n    Version:          1.0.0-rc6+dev\n    GitCommit:        2b18fe1d885ee5083ef9f0838fee39b62d653e30\n   docker-init:\n    Version:          0.18.0\n    GitCommit:        fec3683 ([#77495](https://github.com/kubernetes/kubernetes/pull/77495), [@wanghaoran1988](https://github.com/wanghaoran1988))",
-    "author": "wanghaoran1988",
-    "author_url": "https://github.com/wanghaoran1988",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77495",
-    "pr_number": 77495,
-    "kinds": ["bug"],
-    "sigs": ["testing"],
-    "release_version": "1.14.2"
-  },
-  "77516": {
-    "commit": "3e7fa617b30963ba4fe8e183a1a2a640fb821b69",
-    "text": "Add a field to store CSI volume expansion secrets",
-    "markdown": "Add a field to store CSI volume expansion secrets ([#77516](https://github.com/kubernetes/kubernetes/pull/77516), [@gnufied](https://github.com/gnufied))",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77516",
-    "pr_number": 77516,
-    "kinds": ["api-change"],
-    "sigs": ["apps", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "77528": {
-    "commit": "851afa0beaac20757ad2556af94e31dce0a918be",
-    "text": "Update to use go 1.12.5",
-    "markdown": "Update to use go 1.12.5 ([#77528](https://github.com/kubernetes/kubernetes/pull/77528), [@cblecker](https://github.com/cblecker))",
-    "author": "cblecker",
-    "author_url": "https://github.com/cblecker",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77528",
-    "pr_number": 77528,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["release", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "77559": {
-    "commit": "c54b664b1c937555f80b5ed70092c1676a971d51",
-    "text": "Implement Permit extension point of the scheduling framework.",
-    "markdown": "Implement Permit extension point of the scheduling framework. ([#77559](https://github.com/kubernetes/kubernetes/pull/77559), [@ahg-g](https://github.com/ahg-g))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "ahg-g",
-    "author_url": "https://github.com/ahg-g",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77559",
-    "pr_number": 77559,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "77598": {
-    "commit": "b9ccdd2824c8df7711d3aed985d3e1b549a40de3",
-    "text": "Add Un-reserve extension point for the scheduling framework.",
-    "markdown": "Add Un-reserve extension point for the scheduling framework. ([#77598](https://github.com/kubernetes/kubernetes/pull/77598), [@danielqsj](https://github.com/danielqsj))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77598",
-    "pr_number": 77598,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.2"
-  },
-  "77617": {
-    "commit": "33f44a29b829a8715cd6246935edfd82969e67f9",
-    "text": "ingress:  Update in-tree Ingress controllers, examples, and clients to target networking.k8s.io/v1beta1",
-    "markdown": "ingress:  Update in-tree Ingress controllers, examples, and clients to target networking.k8s.io/v1beta1 ([#77617](https://github.com/kubernetes/kubernetes/pull/77617), [@cmluciano](https://github.com/cmluciano))",
-    "author": "cmluciano",
-    "author_url": "https://github.com/cmluciano",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77617",
-    "pr_number": 77617,
-    "areas": ["e2e-test-framework", "kubectl", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli", "network", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.2"
-  },
-  "77619": {
-    "commit": "ed4c50852452a30369bdf7004b1c06043caf8c92",
-    "text": "Fixed a bug in the apiserver storage that could cause just-added finalizers to be ignored on an immediately following delete request, leading to premature deletion.",
-    "markdown": "Fixed a bug in the apiserver storage that could cause just-added finalizers to be ignored on an immediately following delete request, leading to premature deletion. ([#77619](https://github.com/kubernetes/kubernetes/pull/77619), [@caesarxuchao](https://github.com/caesarxuchao))",
-    "author": "caesarxuchao",
-    "author_url": "https://github.com/caesarxuchao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77619",
-    "pr_number": 77619,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.2"
-  },
-  "77630": {
-    "commit": "e9af72c6e9524ed85c827298f87220e324444d36",
-    "text": "Fix public IPs issues when multiple clusters are sharing the same resource group.\n\n\n\n- If the cluster is upgraded from old releases and the same resource group would be shared by multiple clusters, please recreate those LoadBalancer services or add a new tag 'kubernetes-cluster-name: \u003ccluster-name\u003e' manually for existing public IPs.\n- For multiple clusters sharing the same resource group, they should be configured with different cluster name by `kube-controller-manager --cluster-name=\u003ccluster-name\u003e`",
-    "markdown": "Fix public IPs issues when multiple clusters are sharing the same resource group.\n  \n  \n  \n  - If the cluster is upgraded from old releases and the same resource group would be shared by multiple clusters, please recreate those LoadBalancer services or add a new tag 'kubernetes-cluster-name: \u003ccluster-name\u003e' manually for existing public IPs.\n  - For multiple clusters sharing the same resource group, they should be configured with different cluster name by `kube-controller-manager --cluster-name=\u003ccluster-name\u003e` ([#77630](https://github.com/kubernetes/kubernetes/pull/77630), [@feiskyer](https://github.com/feiskyer))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77630",
-    "pr_number": 77630,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "action_required": true,
-    "release_version": "1.14.2"
-  },
-  "77653": {
-    "commit": "fb41b7a801538c6c8c9385dc35bf1eaae92f06ab",
-    "text": "CRD validation schemas should not specify `metadata` fields other than `name` and `generateName`. A schema will not be considered structural (and therefore ready for future features) if `metadata` is specified in any other way.",
-    "markdown": "CRD validation schemas should not specify `metadata` fields other than `name` and `generateName`. A schema will not be considered structural (and therefore ready for future features) if `metadata` is specified in any other way. ([#77653](https://github.com/kubernetes/kubernetes/pull/77653), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77653",
-    "pr_number": 77653,
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.2"
-  },
-  "77656": {
-    "commit": "ef9e794a369d25f4024317dc5ca5bc04c18f887e",
-    "text": "Check if container memory stats are available before accessing it",
-    "markdown": "Check if container memory stats are available before accessing it ([#77656](https://github.com/kubernetes/kubernetes/pull/77656), [@yastij](https://github.com/yastij))",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77656",
-    "pr_number": 77656,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.2"
-  },
-  "77704": {
-    "commit": "32b37f5a85e8f2987f8367e611147cd4d4dfa4f9",
-    "text": "Once merged, `legacy cloud providers` unit tests will run as part of ci, just as they were before they move from `./pkg/cloudproviders/providers`",
-    "markdown": "Once merged, `legacy cloud providers` unit tests will run as part of ci, just as they were before they move from `./pkg/cloudproviders/providers` ([#77704](https://github.com/kubernetes/kubernetes/pull/77704), [@khenidak](https://github.com/khenidak))",
-    "author": "khenidak",
-    "author_url": "https://github.com/khenidak",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77704",
-    "pr_number": 77704,
-    "kinds": ["cleanup"],
-    "sigs": ["testing"],
-    "release_version": "1.14.2"
-  },
-  "77719": {
-    "commit": "bf79c7c7eedf2e79a721259cf0cbf519cdea623c",
-    "text": "Fix some service tags not supported issues for Azure LoadBalancer service",
-    "markdown": "Fix some service tags not supported issues for Azure LoadBalancer service ([#77719](https://github.com/kubernetes/kubernetes/pull/77719), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77719",
-    "pr_number": 77719,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.2"
-  },
-  "77722": {
-    "commit": "3bc595e37c144882ed4c2a23b8bef4ba82cfffcd",
-    "text": "fix incorrect prometheus azure metrics",
-    "markdown": "fix incorrect prometheus azure metrics ([#77722](https://github.com/kubernetes/kubernetes/pull/77722), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77722",
-    "pr_number": 77722,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.2"
-  }
-}
diff --git a/src/assets/release-notes-1.14.3.json b/src/assets/release-notes-1.14.3.json
deleted file mode 100644
index 2ee798ff..00000000
--- a/src/assets/release-notes-1.14.3.json
+++ /dev/null
@@ -1,1493 +0,0 @@
-{
-  "62159": {
-    "commit": "ec89794c843751899547a685f6579bca86e52e59",
-    "text": "fix bug for regular empty file",
-    "markdown": "fix bug for regular empty file ([#62159](https://github.com/kubernetes/kubernetes/pull/62159), [@dixudx](https://github.com/dixudx))",
-    "author": "dixudx",
-    "author_url": "https://github.com/dixudx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/62159",
-    "pr_number": 62159,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.3"
-  },
-  "65782": {
-    "commit": "e67c266a72bf7b379a2b283a9989804f2282bac5",
-    "text": "Implementing deduplication logic for v1beta1.Event API",
-    "markdown": "Implementing deduplication logic for v1beta1.Event API ([#65782](https://github.com/kubernetes/kubernetes/pull/65782), [@yastij](https://github.com/yastij))",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/65782",
-    "pr_number": 65782,
-    "areas": ["apiserver", "dependency", "release-eng"],
-    "sigs": ["api-machinery", "release"],
-    "duplicate": true,
-    "release_version": "1.14.3"
-  },
-  "66928": {
-    "commit": "cf76868b3430be015cc1e8443a216bf863a9b9f7",
-    "text": "kubelet now allows use of XFS quotas (on XFS and suitably configured ext4fs filesystems) to monitor storage consumption for ephemeral storage (currently for emptydir volumes only).  This method of monitoring consumption is faster and more accurate than the old method of walking the filesystem tree.  It does not enforce limits, only monitors consumption.  To utilize this functionality, you must set the feature  gate `LocalStorageCapacityIsolationFSQuotaMonitoring=true`. For ext4fs filesystems, you must create the\nfilesystem with `mkfs.ext4 -O project \u003cblock_device\u003e` and run `tune2fs -Q prjquota `block device`; XFS\nfilesystems need no additional preparation.  The filesystem must be mounted with option `project` in\n`/etc/fstab`.  If your primary partition is the root filesystem, you must also add `rootflags=pquota` to your\nGRUB config file.",
-    "markdown": "kubelet now allows use of XFS quotas (on XFS and suitably configured ext4fs filesystems) to monitor storage consumption for ephemeral storage (currently for emptydir volumes only).  This method of monitoring consumption is faster and more accurate than the old method of walking the filesystem tree.  It does not enforce limits, only monitors consumption.  To utilize this functionality, you must set the feature  gate `LocalStorageCapacityIsolationFSQuotaMonitoring=true`. For ext4fs filesystems, you must create the\n  filesystem with `mkfs.ext4 -O project \u003cblock_device\u003e` and run `tune2fs -Q prjquota `block device`; XFS\n  filesystems need no additional preparation.  The filesystem must be mounted with option `project` in\n  `/etc/fstab`.  If your primary partition is the root filesystem, you must also add `rootflags=pquota` to your\n  GRUB config file. ([#66928](https://github.com/kubernetes/kubernetes/pull/66928), [@RobertKrawitz](https://github.com/RobertKrawitz))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG CLI, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Node, SIG Release, SIG Scheduling, SIG Storage, and SIG Testing",
-    "author": "RobertKrawitz",
-    "author_url": "https://github.com/RobertKrawitz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/66928",
-    "pr_number": 66928,
-    "areas": ["apiserver", "kubeadm", "kubectl", "kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "node",
-      "release",
-      "scheduling",
-      "storage",
-      "testing"
-    ],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "69867": {
-    "commit": "de6ce5ee16895ce8aef0f0f6faa194541b7596c2",
-    "text": "Allow updates/patches to pod disruption budgets",
-    "markdown": "Allow updates/patches to pod disruption budgets ([#69867](https://github.com/kubernetes/kubernetes/pull/69867), [@davidmccormick](https://github.com/davidmccormick))\n\n  Courtesy of SIG Apps, SIG Auth, and SIG Testing",
-    "author": "davidmccormick",
-    "author_url": "https://github.com/davidmccormick",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/69867",
-    "pr_number": 69867,
-    "areas": ["apiserver", "test"],
-    "kinds": ["feature"],
-    "sigs": ["apps", "auth", "testing"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "70715": {
-    "commit": "3c3c1b11cb4afdfecc73942299243b64b5116acf",
-    "text": "The transformer_failures_total metric is deprecated in favor of transformation_operation_total. The old metric will continue to be populated but will be removed in a future release.",
-    "markdown": "The transformer_failures_total metric is deprecated in favor of transformation_operation_total. The old metric will continue to be populated but will be removed in a future release. ([#70715](https://github.com/kubernetes/kubernetes/pull/70715), [@immutableT](https://github.com/immutableT))\n\n  Courtesy of SIG API Machinery, and SIG Instrumentation",
-    "author": "immutableT",
-    "author_url": "https://github.com/immutableT",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/70715",
-    "pr_number": 70715,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "instrumentation"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "72787": {
-    "commit": "f49fe2a7504c40684573f3e4928e8b0bfdfb2b30",
-    "text": "The kubelet only collects metrics for the node, container runtime, kubelet, pods, and containers.",
-    "markdown": "The kubelet only collects metrics for the node, container runtime, kubelet, pods, and containers. ([#72787](https://github.com/kubernetes/kubernetes/pull/72787), [@dashpole](https://github.com/dashpole))\n\n  Courtesy of SIG Node, and SIG Testing",
-    "author": "dashpole",
-    "author_url": "https://github.com/dashpole",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72787",
-    "pr_number": 72787,
-    "areas": ["kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["node", "testing"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "73241": {
-    "commit": "e476a60ccbe25581f5a6a9401081dcee311a066e",
-    "text": "Set selinux label at plugin socket directory",
-    "markdown": "Set selinux label at plugin socket directory ([#73241](https://github.com/kubernetes/kubernetes/pull/73241), [@vikaschoudhary16](https://github.com/vikaschoudhary16))",
-    "author": "vikaschoudhary16",
-    "author_url": "https://github.com/vikaschoudhary16",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73241",
-    "pr_number": 73241,
-    "areas": ["hw-accelerators", "kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.3"
-  },
-  "73447": {
-    "commit": "9577cbfb9fd607d3698e1f2440f5875c85eca384",
-    "text": "Improved README for k8s.io/sample-apiserver",
-    "markdown": "Improved README for k8s.io/sample-apiserver ([#73447](https://github.com/kubernetes/kubernetes/pull/73447), [@MikeSpreitzer](https://github.com/MikeSpreitzer))",
-    "author": "MikeSpreitzer",
-    "author_url": "https://github.com/MikeSpreitzer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73447",
-    "pr_number": 73447,
-    "kinds": ["documentation"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.3"
-  },
-  "73891": {
-    "commit": "fe37733a124766ee1da11cf1a173488788845cb2",
-    "text": "Kubelet plugin registration now has retry and exponential backoff logic for when registration of plugins (like CSI or device plugin) fail.",
-    "markdown": "Kubelet plugin registration now has retry and exponential backoff logic for when registration of plugins (like CSI or device plugin) fail. ([#73891](https://github.com/kubernetes/kubernetes/pull/73891), [@taragu](https://github.com/taragu))\n\n  Courtesy of SIG Node, and SIG Storage",
-    "author": "taragu",
-    "author_url": "https://github.com/taragu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73891",
-    "pr_number": 73891,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node", "storage"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "74027": {
-    "commit": "74743793f2b4de7a2c5e02478ee9dcf7a360ed0a",
-    "text": "The metric `kube_proxy_sync_proxy_rules_last_timestamp_seconds` is now available, indicating the last time that kube-proxy successfully applied proxying rules.",
-    "markdown": "The metric `kube_proxy_sync_proxy_rules_last_timestamp_seconds` is now available, indicating the last time that kube-proxy successfully applied proxying rules. ([#74027](https://github.com/kubernetes/kubernetes/pull/74027), [@squeed](https://github.com/squeed))\n\n  Courtesy of SIG Instrumentation, and SIG Network",
-    "author": "squeed",
-    "author_url": "https://github.com/squeed",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74027",
-    "pr_number": 74027,
-    "kinds": ["feature"],
-    "sigs": ["instrumentation", "network"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "74614": {
-    "commit": "6c42729c2c01b2792fa5f5c95d825645eaba2571",
-    "text": "Add  NonPrempting field to the PriorityClass.",
-    "markdown": "Add  NonPrempting field to the PriorityClass. ([#74614](https://github.com/kubernetes/kubernetes/pull/74614), [@denkensk](https://github.com/denkensk))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG CLI, SIG Node, SIG Scheduling, and SIG Testing",
-    "author": "denkensk",
-    "author_url": "https://github.com/denkensk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74614",
-    "pr_number": 74614,
-    "areas": ["apiserver", "kubectl", "kubelet", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "auth", "cli", "node", "scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "74692": {
-    "commit": "009f7a07ff4f793c56c0d66d213c54998624f31e",
-    "text": "refactor AWS NLB securityGroup handling",
-    "markdown": "refactor AWS NLB securityGroup handling ([#74692](https://github.com/kubernetes/kubernetes/pull/74692), [@M00nF1sh](https://github.com/M00nF1sh))",
-    "author": "M00nF1sh",
-    "author_url": "https://github.com/M00nF1sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74692",
-    "pr_number": 74692,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.3"
-  },
-  "74825": {
-    "commit": "8b7e777fe8843419db21293759be861b5a4e682b",
-    "text": "Windows support for preserving the destination IP as the VIP when loadbalancing with DSR.",
-    "markdown": "Windows support for preserving the destination IP as the VIP when loadbalancing with DSR. ([#74825](https://github.com/kubernetes/kubernetes/pull/74825), [@ksubrmnn](https://github.com/ksubrmnn))\n\n  Courtesy of SIG Network, and SIG Windows",
-    "author": "ksubrmnn",
-    "author_url": "https://github.com/ksubrmnn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74825",
-    "pr_number": 74825,
-    "areas": ["dependency"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["network", "windows"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "75109": {
-    "commit": "8a822b0fcf394e58cb6406d4194eb5237d3a333f",
-    "text": "Allow init container to get its own field value as environment variable values(downwardAPI spport)",
-    "markdown": "Allow init container to get its own field value as environment variable values(downwardAPI spport) ([#75109](https://github.com/kubernetes/kubernetes/pull/75109), [@yuchengwu](https://github.com/yuchengwu))",
-    "author": "yuchengwu",
-    "author_url": "https://github.com/yuchengwu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75109",
-    "pr_number": 75109,
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.3"
-  },
-  "75405": {
-    "commit": "ec88ec272ccdc824c56684628b113f09b873a9b2",
-    "text": "Fix bug where cloud-controller-manager initializes nodes multiple times",
-    "markdown": "Fix bug where cloud-controller-manager initializes nodes multiple times ([#75405](https://github.com/kubernetes/kubernetes/pull/75405), [@tghartland](https://github.com/tghartland))",
-    "author": "tghartland",
-    "author_url": "https://github.com/tghartland",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75405",
-    "pr_number": 75405,
-    "areas": ["cloudprovider", "controller-manager"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "cloud-provider", "scalability"],
-    "duplicate": true,
-    "release_version": "1.14.3"
-  },
-  "75459": {
-    "commit": "ae2a1628f66f68a59625283f427f0a794ad0d53a",
-    "text": "API changes and deprecating the use of special annotations for Windows GMSA support (version beta)",
-    "markdown": "API changes and deprecating the use of special annotations for Windows GMSA support (version beta) ([#75459](https://github.com/kubernetes/kubernetes/pull/75459), [@wk8](https://github.com/wk8))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Node, SIG Testing, and SIG Windows",
-    "author": "wk8",
-    "author_url": "https://github.com/wk8",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75459",
-    "pr_number": 75459,
-    "areas": ["kubelet", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "node", "testing", "windows"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "75479": {
-    "commit": "89ae84d478d418b1bf4b44c4544020ab110c5bf5",
-    "text": "Kubelet will attempt to use wincat.exe in the pause container for port forwarding when running on Windows",
-    "markdown": "Kubelet will attempt to use wincat.exe in the pause container for port forwarding when running on Windows ([#75479](https://github.com/kubernetes/kubernetes/pull/75479), [@benmoss](https://github.com/benmoss))\n\n  Courtesy of SIG Node, and SIG Windows",
-    "author": "benmoss",
-    "author_url": "https://github.com/benmoss",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75479",
-    "pr_number": 75479,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node", "windows"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "75499": {
-    "commit": "73b8011d4744be6465ecc762cc84d3cbc092d1f9",
-    "text": "kubeadm's ignored pre-flight errors can now be configured via InitConfiguration and JoinConfiguration.",
-    "markdown": "kubeadm's ignored pre-flight errors can now be configured via InitConfiguration and JoinConfiguration. ([#75499](https://github.com/kubernetes/kubernetes/pull/75499), [@marccarre](https://github.com/marccarre))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "marccarre",
-    "author_url": "https://github.com/marccarre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75499",
-    "pr_number": 75499,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "75920": {
-    "commit": "3d871df19a1686c1f9a1e44ec0aab207d61805be",
-    "text": "Add --sort-by option to kubectl top command",
-    "markdown": "Add --sort-by option to kubectl top command ([#75920](https://github.com/kubernetes/kubernetes/pull/75920), [@artmello](https://github.com/artmello))\n\n  Courtesy of SIG CLI",
-    "author": "artmello",
-    "author_url": "https://github.com/artmello",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75920",
-    "pr_number": 75920,
-    "areas": ["kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "75990": {
-    "commit": "4036097edb8c77ca553d0b6901926db7b6447798",
-    "text": "consume the AWS region list from the AWS SDK instead of a hard-coded list in the cloud provider",
-    "markdown": "consume the AWS region list from the AWS SDK instead of a hard-coded list in the cloud provider ([#75990](https://github.com/kubernetes/kubernetes/pull/75990), [@mcrute](https://github.com/mcrute))",
-    "author": "mcrute",
-    "author_url": "https://github.com/mcrute",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75990",
-    "pr_number": 75990,
-    "areas": ["cloudprovider"],
-    "kinds": ["cleanup"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.3"
-  },
-  "76188": {
-    "commit": "fad34e404ef690102d2a86b939a8a581e976e5ae",
-    "text": "Expose CSI volume stats via kubelet volume metrics",
-    "markdown": "Expose CSI volume stats via kubelet volume metrics ([#76188](https://github.com/kubernetes/kubernetes/pull/76188), [@humblec](https://github.com/humblec))\n\n  Courtesy of SIG Storage",
-    "author": "humblec",
-    "author_url": "https://github.com/humblec",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76188",
-    "pr_number": 76188,
-    "kinds": ["feature"],
-    "sigs": ["storage"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "76551": {
-    "commit": "7929c15ec7a9cb27272b7ecb098f13ef135ebde4",
-    "text": "proxy/transport: Support Content-Encoding: deflate",
-    "markdown": "proxy/transport: Support Content-Encoding: deflate ([#76551](https://github.com/kubernetes/kubernetes/pull/76551), [@JieJhih](https://github.com/JieJhih))\n\n  Courtesy of SIG API Machinery",
-    "author": "JieJhih",
-    "author_url": "https://github.com/JieJhih",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76551",
-    "pr_number": 76551,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "76713": {
-    "commit": "48f2be9cff605b79069fc7aa5795bebc2d83c495",
-    "text": "Remove deprecated --pod/-p flag from kubectl exec. The flag has been marked as deprecated since k8s version v1.12",
-    "markdown": "Remove deprecated --pod/-p flag from kubectl exec. The flag has been marked as deprecated since k8s version v1.12 ([#76713](https://github.com/kubernetes/kubernetes/pull/76713), [@prksu](https://github.com/prksu))",
-    "author": "prksu",
-    "author_url": "https://github.com/prksu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76713",
-    "pr_number": 76713,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.14.3"
-  },
-  "76913": {
-    "commit": "82bfa667ed543b3fdf79f51a22389e2ba594da33",
-    "text": "This change enables a user to specify a DataSource/Kind of type \"PersistentVolumeClaim\" in their PVC spec.  This can then be detected by the external csi-provisioner and plugins if capable.",
-    "markdown": "This change enables a user to specify a DataSource/Kind of type \"PersistentVolumeClaim\" in their PVC spec.  This can then be detected by the external csi-provisioner and plugins if capable. ([#76913](https://github.com/kubernetes/kubernetes/pull/76913), [@j-griffith](https://github.com/j-griffith))",
-    "author": "j-griffith",
-    "author_url": "https://github.com/j-griffith",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76913",
-    "pr_number": 76913,
-    "kinds": ["api-change"],
-    "sigs": ["storage"],
-    "release_version": "1.14.3"
-  },
-  "76915": {
-    "commit": "3aecf4350e3b518e8786df7422b9036f57832eb5",
-    "text": "Inline CSI ephemeral volumes can now be controlled with PodSecurityPolicy when the CSIInlineVolume alpha feature is enabled",
-    "markdown": "Inline CSI ephemeral volumes can now be controlled with PodSecurityPolicy when the CSIInlineVolume alpha feature is enabled ([#76915](https://github.com/kubernetes/kubernetes/pull/76915), [@vladimirvivien](https://github.com/vladimirvivien))",
-    "author": "vladimirvivien",
-    "author_url": "https://github.com/vladimirvivien",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76915",
-    "pr_number": 76915,
-    "areas": ["kubectl"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["auth", "cli", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.3"
-  },
-  "77170": {
-    "commit": "4891eaa3adbfafb61d2bd264d2f0daef124ee8b9",
-    "text": "DelayingQueue.ShutDown() is now able to be invoked multiple times without causing a closed channel panic.",
-    "markdown": "DelayingQueue.ShutDown() is now able to be invoked multiple times without causing a closed channel panic. ([#77170](https://github.com/kubernetes/kubernetes/pull/77170), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77170",
-    "pr_number": 77170,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.3"
-  },
-  "77274": {
-    "commit": "fc005789c1162e55b22fb118dceaec5c0622b1db",
-    "text": "Enable 3rd party device monitoring by default",
-    "markdown": "Enable 3rd party device monitoring by default ([#77274](https://github.com/kubernetes/kubernetes/pull/77274), [@RenaudWasTaken](https://github.com/RenaudWasTaken))\n\n  Courtesy of SIG Node",
-    "author": "RenaudWasTaken",
-    "author_url": "https://github.com/RenaudWasTaken",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77274",
-    "pr_number": 77274,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "77333": {
-    "commit": "3193e78a2f2d2296de0f16a1da3509b4b7ef7a49",
-    "text": "Add CRD spec.preserveUnknownFields boolean, defaulting to true in v1beta1 and to false in v1 CRDs. If false, fields not specified in the validation schema will be removed when sent to the API server or when read from etcd.",
-    "markdown": "Add CRD spec.preserveUnknownFields boolean, defaulting to true in v1beta1 and to false in v1 CRDs. If false, fields not specified in the validation schema will be removed when sent to the API server or when read from etcd. ([#77333](https://github.com/kubernetes/kubernetes/pull/77333), [@sttts](https://github.com/sttts))\n\n  Courtesy of SIG API Machinery, and SIG Testing",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77333",
-    "pr_number": 77333,
-    "areas": ["apiserver", "custom-resources", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "testing"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "77371": {
-    "commit": "aa25195ab1f046fcdf8886ef1b6cecdd8e1088bd",
-    "text": "Fix a bug where kube-proxy returns error due to existing ipset rules using a different hash type.",
-    "markdown": "Fix a bug where kube-proxy returns error due to existing ipset rules using a different hash type. ([#77371](https://github.com/kubernetes/kubernetes/pull/77371), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77371",
-    "pr_number": 77371,
-    "areas": ["ipvs"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.3"
-  },
-  "77423": {
-    "commit": "6dfc9961567ae361461f7fc91083ff7d80b617c7",
-    "text": "`kubectl rollout restart` now works for daemonsets and statefulsets.",
-    "markdown": "`kubectl rollout restart` now works for daemonsets and statefulsets. ([#77423](https://github.com/kubernetes/kubernetes/pull/77423), [@apelisse](https://github.com/apelisse))\n\n  Courtesy of SIG CLI, and SIG Testing",
-    "author": "apelisse",
-    "author_url": "https://github.com/apelisse",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77423",
-    "pr_number": 77423,
-    "areas": ["kubectl", "test"],
-    "kinds": ["feature"],
-    "sigs": ["cli", "testing"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "77475": {
-    "commit": "6feea43bcc75895534d5ba169b711c8cacce4dae",
-    "text": "When the number of jobs exceeds 500, cronjob should schedule without error.",
-    "markdown": "When the number of jobs exceeds 500, cronjob should schedule without error. ([#77475](https://github.com/kubernetes/kubernetes/pull/77475), [@liucimin](https://github.com/liucimin))",
-    "author": "liucimin",
-    "author_url": "https://github.com/liucimin",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77475",
-    "pr_number": 77475,
-    "kinds": ["bug"],
-    "sigs": ["apps", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.14.3"
-  },
-  "77501": {
-    "commit": "df4f033a4f0c5103cd703fedcc91b16bfdb33106",
-    "text": "Add configuration options for the scheduling framework and its plugins.",
-    "markdown": "Add configuration options for the scheduling framework and its plugins. ([#77501](https://github.com/kubernetes/kubernetes/pull/77501), [@JieJhih](https://github.com/JieJhih))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "JieJhih",
-    "author_url": "https://github.com/JieJhih",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77501",
-    "pr_number": 77501,
-    "areas": ["test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "77507": {
-    "commit": "c854f72dcba5ca465d978e6e01ebffdaa4c9d2c4",
-    "text": "kubeadm: fix a bug related to volume unmount if the kubelet run directory is a symbolic link",
-    "markdown": "kubeadm: fix a bug related to volume unmount if the kubelet run directory is a symbolic link ([#77507](https://github.com/kubernetes/kubernetes/pull/77507), [@cuericlee](https://github.com/cuericlee))",
-    "author": "cuericlee",
-    "author_url": "https://github.com/cuericlee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77507",
-    "pr_number": 77507,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.3"
-  },
-  "77523": {
-    "commit": "bdf3d248eb401f48bb0ed5a159cb4264803dd789",
-    "text": "iptables proxier: route local traffic to LB IPs to service chain",
-    "markdown": "iptables proxier: route local traffic to LB IPs to service chain ([#77523](https://github.com/kubernetes/kubernetes/pull/77523), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77523",
-    "pr_number": 77523,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.3"
-  },
-  "77529": {
-    "commit": "796ecb939183ed70f4b8508cfc5bee3fba507b38",
-    "text": "Support \"queue-sort\" extension point for scheduling framework",
-    "markdown": "Support \"queue-sort\" extension point for scheduling framework ([#77529](https://github.com/kubernetes/kubernetes/pull/77529), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Scheduling",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77529",
-    "pr_number": 77529,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "77563": {
-    "commit": "0b6ad8bc3f0e85e3a0f1a8a94da3ca4068506de3",
-    "text": "Add `Option` field to the admission webhook `AdmissionReview` API that provides the operation options (e.g. `DeleteOption` or `CreateOption`) for the operation being performed.",
-    "markdown": "Add `Option` field to the admission webhook `AdmissionReview` API that provides the operation options (e.g. `DeleteOption` or `CreateOption`) for the operation being performed. ([#77563](https://github.com/kubernetes/kubernetes/pull/77563), [@jpbetz](https://github.com/jpbetz))\n\n  Courtesy of SIG API Machinery, SIG Auth, and SIG Testing",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77563",
-    "pr_number": 77563,
-    "areas": ["admission-control", "apiserver", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "auth", "testing"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "77567": {
-    "commit": "bb475c4a645afa96a648cd90342e70d2884206c3",
-    "text": "Add Post-bind extension point to the scheduling framework",
-    "markdown": "Add Post-bind extension point to the scheduling framework ([#77567](https://github.com/kubernetes/kubernetes/pull/77567), [@wgliang](https://github.com/wgliang))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77567",
-    "pr_number": 77567,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "77589": {
-    "commit": "74469297fe75b3177aec099a2d794333d4e054c7",
-    "text": "Fix kubectl exec usage string",
-    "markdown": "Fix kubectl exec usage string ([#77589](https://github.com/kubernetes/kubernetes/pull/77589), [@soltysh](https://github.com/soltysh))",
-    "author": "soltysh",
-    "author_url": "https://github.com/soltysh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77589",
-    "pr_number": 77589,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.14.3"
-  },
-  "77611": {
-    "commit": "6e78282a822e5b4aaf974c3430ff578f0d7ababc",
-    "text": "kube-apiserver: the `--enable-logs-handler` flag and log-serving functionality is deprecated, and scheduled to be removed in v1.19.",
-    "markdown": "kube-apiserver: the `--enable-logs-handler` flag and log-serving functionality is deprecated, and scheduled to be removed in v1.19. ([#77611](https://github.com/kubernetes/kubernetes/pull/77611), [@rohitsardesai83](https://github.com/rohitsardesai83))",
-    "author": "rohitsardesai83",
-    "author_url": "https://github.com/rohitsardesai83",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77611",
-    "pr_number": 77611,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.3"
-  },
-  "77739": {
-    "commit": "6a0db7b1894fd7c993c761ccaea1d3984ed70f7f",
-    "text": "kubeadm: v1beta2 InitConfiguration no longer embeds ClusterConfiguration it it.",
-    "markdown": "kubeadm: v1beta2 InitConfiguration no longer embeds ClusterConfiguration it it. ([#77739](https://github.com/kubernetes/kubernetes/pull/77739), [@rosti](https://github.com/rosti))",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77739",
-    "pr_number": 77739,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.3"
-  },
-  "77743": {
-    "commit": "358bfe3808c1d3d13e63f56de22c31c8ae5e859d",
-    "text": "In CRD webhook conversion ignore changes to metadata other than for labels and annotations.",
-    "markdown": "In CRD webhook conversion ignore changes to metadata other than for labels and annotations. ([#77743](https://github.com/kubernetes/kubernetes/pull/77743), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77743",
-    "pr_number": 77743,
-    "areas": ["custom-resources", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.3"
-  },
-  "77755": {
-    "commit": "99f2e78c9a5673bf0f5db1b636d6307352946f29",
-    "text": "Move online volume expansion to beta",
-    "markdown": "Move online volume expansion to beta ([#77755](https://github.com/kubernetes/kubernetes/pull/77755), [@gnufied](https://github.com/gnufied))\n\n  Courtesy of SIG Node, SIG Storage, and SIG Testing",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77755",
-    "pr_number": 77755,
-    "areas": ["kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["node", "storage", "testing"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "77773": {
-    "commit": "6ba13bf151290ae3fd3c6acfd017e233f21ccd59",
-    "text": "Fix a bug that causes DaemonSet rolling update to hang when its pod gets stuck at terminating.",
-    "markdown": "Fix a bug that causes DaemonSet rolling update to hang when its pod gets stuck at terminating. ([#77773](https://github.com/kubernetes/kubernetes/pull/77773), [@DaiHao](https://github.com/DaiHao))",
-    "author": "DaiHao",
-    "author_url": "https://github.com/DaiHao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77773",
-    "pr_number": 77773,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.14.3"
-  },
-  "77780": {
-    "commit": "e1770e698e97437d0fdf0a8a0351406cf6c135d1",
-    "text": "kubeadm: kubeadm alpha certs renew  --csr-only now reads the current certificates as the authoritative source for certificates attributes (same as kubeadm alpha certs renew)",
-    "markdown": "kubeadm: kubeadm alpha certs renew  --csr-only now reads the current certificates as the authoritative source for certificates attributes (same as kubeadm alpha certs renew) ([#77780](https://github.com/kubernetes/kubernetes/pull/77780), [@fabriziopandini](https://github.com/fabriziopandini))",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77780",
-    "pr_number": 77780,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.3"
-  },
-  "77802": {
-    "commit": "944a7e2be685aade95202aba38b6683ba399963f",
-    "text": "IPVS: Disable graceful termination for UDP traffic to solve issues with high number of UDP connections (DNS / syslog in particular)",
-    "markdown": "IPVS: Disable graceful termination for UDP traffic to solve issues with high number of UDP connections (DNS / syslog in particular) ([#77802](https://github.com/kubernetes/kubernetes/pull/77802), [@lbernail](https://github.com/lbernail))",
-    "author": "lbernail",
-    "author_url": "https://github.com/lbernail",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77802",
-    "pr_number": 77802,
-    "areas": ["ipvs"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.3"
-  },
-  "77816": {
-    "commit": "c85c0e4780e428fdfb33debc31bc88f8095e5831",
-    "text": "Fixed a transient error API requests for custom resources could encounter while changes to the CustomResourceDefinition were being applied.",
-    "markdown": "Fixed a transient error API requests for custom resources could encounter while changes to the CustomResourceDefinition were being applied. ([#77816](https://github.com/kubernetes/kubernetes/pull/77816), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77816",
-    "pr_number": 77816,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.3"
-  },
-  "77820": {
-    "commit": "4f33b5f42146400c9aa0a1f0f5ad79607ec06f6d",
-    "text": "Deprecated Kubelet security controls AllowPrivileged, HostNetworkSources, HostPIDSources, HostIPCSources have been removed. Enforcement of these restrictions should be done through admission control instead (e.g. PodSecurityPolicy).\nThe deprecated Kubelet flag `--allow-privileged` has been removed. Remove any use of `--allow-privileged` from your kubelet scripts or manifests.",
-    "markdown": "Deprecated Kubelet security controls AllowPrivileged, HostNetworkSources, HostPIDSources, HostIPCSources have been removed. Enforcement of these restrictions should be done through admission control instead (e.g. PodSecurityPolicy).\n  The deprecated Kubelet flag `--allow-privileged` has been removed. Remove any use of `--allow-privileged` from your kubelet scripts or manifests. ([#77820](https://github.com/kubernetes/kubernetes/pull/77820), [@dims](https://github.com/dims))\n\n  Courtesy of SIG Cluster Lifecycle, SIG Node, and SIG Testing",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77820",
-    "pr_number": 77820,
-    "areas": ["kubelet", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "node", "testing"],
-    "action_required": true,
-    "release_version": "1.14.3"
-  },
-  "77825": {
-    "commit": "a07b027261383127494fc6d4e3ca2c544fb4a464",
-    "text": "The CustomResourcePublishOpenAPI feature is now beta and enabled by default. CustomResourceDefinitions with [structural schemas](https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190425-structural-openapi.md) now publish schemas in the OpenAPI document served at `/openapi/v2`. CustomResourceDefinitions with non-structural schemas have a `NonStructuralSchema` condition added with details about what needs to be corrected in the validation schema.",
-    "markdown": "The CustomResourcePublishOpenAPI feature is now beta and enabled by default. CustomResourceDefinitions with [structural schemas](https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190425-structural-openapi.md) now publish schemas in the OpenAPI document served at `/openapi/v2`. CustomResourceDefinitions with non-structural schemas have a `NonStructuralSchema` condition added with details about what needs to be corrected in the validation schema. ([#77825](https://github.com/kubernetes/kubernetes/pull/77825), [@roycaihw](https://github.com/roycaihw))\n\n  Courtesy of SIG API Machinery, and SIG Testing",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77825",
-    "pr_number": 77825,
-    "areas": ["custom-resources", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "testing"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "77843": {
-    "commit": "76c214b259e2542613e4183eb2c5fa02423c70e8",
-    "text": "Publish DeleteOptions parameters for deletecollection endpoints in OpenAPI spec",
-    "markdown": "Publish DeleteOptions parameters for deletecollection endpoints in OpenAPI spec ([#77843](https://github.com/kubernetes/kubernetes/pull/77843), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77843",
-    "pr_number": 77843,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.3"
-  },
-  "77847": {
-    "commit": "d823fa23c674363c40e9e196f24b88018f04614f",
-    "text": "kubeadm: expose the kubeadm reset command as phases",
-    "markdown": "kubeadm: expose the kubeadm reset command as phases ([#77847](https://github.com/kubernetes/kubernetes/pull/77847), [@yagonobre](https://github.com/yagonobre))",
-    "author": "yagonobre",
-    "author_url": "https://github.com/yagonobre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77847",
-    "pr_number": 77847,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.3"
-  },
-  "77848": {
-    "commit": "6118b8aa735b8ee7245fb851b8fe9199d778e30e",
-    "text": "Add `kubeadm alpha certs certificate-key` command to generate secure random key to use on `kubeadm init --experimental-upload-certs`",
-    "markdown": "Add `kubeadm alpha certs certificate-key` command to generate secure random key to use on `kubeadm init --experimental-upload-certs` ([#77848](https://github.com/kubernetes/kubernetes/pull/77848), [@yagonobre](https://github.com/yagonobre))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "yagonobre",
-    "author_url": "https://github.com/yagonobre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77848",
-    "pr_number": 77848,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "77851": {
-    "commit": "b5336132336d9a8696001594b716c85084f98e02",
-    "text": "remove vmsizelist call in azure disk GetVolumeLimits which happens in kubelet finally",
-    "markdown": "remove vmsizelist call in azure disk GetVolumeLimits which happens in kubelet finally ([#77851](https://github.com/kubernetes/kubernetes/pull/77851), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77851",
-    "pr_number": 77851,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.3"
-  },
-  "77863": {
-    "commit": "81a61ae0e37143299ee5947a6c2c5195ec5f72ae",
-    "text": "Kubeadm: a new command `kubeadm alpha certs check-expiration` was created in order to help users in managing expiration for local PKI certificates",
-    "markdown": "Kubeadm: a new command `kubeadm alpha certs check-expiration` was created in order to help users in managing expiration for local PKI certificates ([#77863](https://github.com/kubernetes/kubernetes/pull/77863), [@fabriziopandini](https://github.com/fabriziopandini))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77863",
-    "pr_number": 77863,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "77874": {
-    "commit": "730bc968b95842022c4c81d607bf6411b459a675",
-    "text": "client-go and kubectl no longer write cached discovery files with world-accessible file permissions",
-    "markdown": "client-go and kubectl no longer write cached discovery files with world-accessible file permissions ([#77874](https://github.com/kubernetes/kubernetes/pull/77874), [@yuchengwu](https://github.com/yuchengwu))",
-    "author": "yuchengwu",
-    "author_url": "https://github.com/yuchengwu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77874",
-    "pr_number": 77874,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.3"
-  },
-  "77887": {
-    "commit": "0f78b57fef69c8ec2f3b2f1934d488fea613f6e6",
-    "text": "NodeLocal DNSCache graduating to beta.",
-    "markdown": "NodeLocal DNSCache graduating to beta. ([#77887](https://github.com/kubernetes/kubernetes/pull/77887), [@prameshj](https://github.com/prameshj))",
-    "author": "prameshj",
-    "author_url": "https://github.com/prameshj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77887",
-    "pr_number": 77887,
-    "kinds": ["documentation"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "duplicate": true,
-    "release_version": "1.14.3"
-  },
-  "77888": {
-    "commit": "adf6fa69877a50f6cad4ce24214cbf2d41f705a4",
-    "text": "Fix panic logspam when running kubelet in standalone mode.",
-    "markdown": "Fix panic logspam when running kubelet in standalone mode. ([#77888](https://github.com/kubernetes/kubernetes/pull/77888), [@tallclair](https://github.com/tallclair))",
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77888",
-    "pr_number": 77888,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.3"
-  },
-  "77889": {
-    "commit": "72f69546142a84590550e37d70260639f8fa3e88",
-    "text": "Support starting Kubernetes on GCE using containerd in COS and Ubuntu with `KUBE_CONTAINER_RUNTIME=containerd`.",
-    "markdown": "Support starting Kubernetes on GCE using containerd in COS and Ubuntu with `KUBE_CONTAINER_RUNTIME=containerd`. ([#77889](https://github.com/kubernetes/kubernetes/pull/77889), [@Random-Liu](https://github.com/Random-Liu))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Node",
-    "author": "Random-Liu",
-    "author_url": "https://github.com/Random-Liu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77889",
-    "pr_number": 77889,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "node"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "77906": {
-    "commit": "5a233ccb84002e18cc8fabc107470a14ac5ebbb8",
-    "text": "Kubelet could be run with no Azure identity now. A sample cloud provider configure is:  `{\"vmType\": \"vmss\", \"useInstanceMetadata\": true, \"subscriptionId\": \"\u003csubscriptionId\u003e\"}`",
-    "markdown": "Kubelet could be run with no Azure identity now. A sample cloud provider configure is:  `{\"vmType\": \"vmss\", \"useInstanceMetadata\": true, \"subscriptionId\": \"\u003csubscriptionId\u003e\"}` ([#77906](https://github.com/kubernetes/kubernetes/pull/77906), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77906",
-    "pr_number": 77906,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.3"
-  },
-  "77912": {
-    "commit": "6a9ecf71fdbf9977254d7298c8f073c68683dced",
-    "text": "Fix incorrect azuredisk lun error",
-    "markdown": "Fix incorrect azuredisk lun error ([#77912](https://github.com/kubernetes/kubernetes/pull/77912), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77912",
-    "pr_number": 77912,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.3"
-  },
-  "77945": {
-    "commit": "e0f28e5d0f03392e2970742426438c05a1778cb4",
-    "text": "k8s.io/client-go/dynamic/dynamicinformer.NewFilteredDynamicSharedInformerFactory now honours namespace argument",
-    "markdown": "k8s.io/client-go/dynamic/dynamicinformer.NewFilteredDynamicSharedInformerFactory now honours namespace argument ([#77945](https://github.com/kubernetes/kubernetes/pull/77945), [@michaelfig](https://github.com/michaelfig))",
-    "author": "michaelfig",
-    "author_url": "https://github.com/michaelfig",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77945",
-    "pr_number": 77945,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.3"
-  },
-  "77946": {
-    "commit": "628e0d8333ea4ef04655ddfd4350561eeaec7262",
-    "text": "Deprecates the kubeadm config upload command as it's replacement is now graduated. Please see `kubeadm init phase upload-config`",
-    "markdown": "Deprecates the kubeadm config upload command as it's replacement is now graduated. Please see `kubeadm init phase upload-config` ([#77946](https://github.com/kubernetes/kubernetes/pull/77946), [@Klaven](https://github.com/Klaven))",
-    "author": "Klaven",
-    "author_url": "https://github.com/Klaven",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77946",
-    "pr_number": 77946,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.3"
-  },
-  "77952": {
-    "commit": "27410955e2db0496af0aaa91ac1cdfc8ca20b1fd",
-    "text": "Updates that remove remaining `metadata.finalizers` from  an object that is pending deletion (non-nil metadata.deletionTimestamp) and has no graceful deletion pending (nil or 0 metadata.deletionGracePeriodSeconds) now results in immediate deletion of the object.",
-    "markdown": "Updates that remove remaining `metadata.finalizers` from  an object that is pending deletion (non-nil metadata.deletionTimestamp) and has no graceful deletion pending (nil or 0 metadata.deletionGracePeriodSeconds) now results in immediate deletion of the object. ([#77952](https://github.com/kubernetes/kubernetes/pull/77952), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77952",
-    "pr_number": 77952,
-    "areas": ["apiserver", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "apps", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.3"
-  },
-  "77965": {
-    "commit": "76c112ed06b9085fba86d14e7e62d35dc5efaf66",
-    "text": "--- FAIL: TestWatchBasedManager (0.32s)\n    testserver.go:195: failed to launch server: failed to set default ServerRunOptions: error creating self-signed certificates: unable to generate self signed cert: failed to write cert fixture to cmd/kube-apiserver/app/testing/testdata/127.0.0.1_10.0.0.1_kubernetes.default.svc-kubernetes.default-kubernetes-localhost.crt: open cmd/kube-apiserver/app/testing/testdata/127.0.0.1_10.0.0.1_kubernetes.default.svc-kubernetes.default-kubernetes-localhost.crt: no such file or directory",
-    "markdown": "--- FAIL: TestWatchBasedManager (0.32s)\n      testserver.go:195: failed to launch server: failed to set default ServerRunOptions: error creating self-signed certificates: unable to generate self signed cert: failed to write cert fixture to cmd/kube-apiserver/app/testing/testdata/127.0.0.1_10.0.0.1_kubernetes.default.svc-kubernetes.default-kubernetes-localhost.crt: open cmd/kube-apiserver/app/testing/testdata/127.0.0.1_10.0.0.1_kubernetes.default.svc-kubernetes.default-kubernetes-localhost.crt: no such file or directory ([#77965](https://github.com/kubernetes/kubernetes/pull/77965), [@lubinsz](https://github.com/lubinsz))",
-    "author": "lubinsz",
-    "author_url": "https://github.com/lubinsz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77965",
-    "pr_number": 77965,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["testing"],
-    "release_version": "1.14.3"
-  },
-  "77975": {
-    "commit": "408735e9dab315bd8f4400fdfa7790ccc3f32ffc",
-    "text": "metrics added to kubelet endpoint 'metrics/probes':\n\n   process_start_time_seconds",
-    "markdown": "metrics added to kubelet endpoint 'metrics/probes':\n  \n     process_start_time_seconds ([#77975](https://github.com/kubernetes/kubernetes/pull/77975), [@logicalhan](https://github.com/logicalhan))",
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77975",
-    "pr_number": 77975,
-    "areas": ["dependency", "kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "instrumentation", "node"],
-    "duplicate": true,
-    "release_version": "1.14.3"
-  },
-  "77990": {
-    "commit": "82ac5c3cbf04ca63777e340c3bdf9e527dfe0ad0",
-    "text": "Fixed a scheduler racing issue to ensure low priority pods to be unschedulable on the node(s) where high priority pods have `NominatedNodeName` set to the node(s).",
-    "markdown": "Fixed a scheduler racing issue to ensure low priority pods to be unschedulable on the node(s) where high priority pods have `NominatedNodeName` set to the node(s). ([#77990](https://github.com/kubernetes/kubernetes/pull/77990), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77990",
-    "pr_number": 77990,
-    "kinds": ["bug", "flake"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.3"
-  },
-  "77994": {
-    "commit": "05df640f33232e9898ada362371faeabd7cdf395",
-    "text": "Handle resize operation for volume plugins migrated to CSI",
-    "markdown": "Handle resize operation for volume plugins migrated to CSI ([#77994](https://github.com/kubernetes/kubernetes/pull/77994), [@gnufied](https://github.com/gnufied))\n\n  Courtesy of SIG API Machinery, SIG Apps, and SIG Storage",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77994",
-    "pr_number": 77994,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "apps", "storage"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "78012": {
-    "commit": "961ea6bd7656302339632c1d4fa5264bcfd52b27",
-    "text": "Upgrade Azure network API version to 2018-07-01, so that EnableTcpReset could be enabled on Azure standard loadbalancer (SLB).",
-    "markdown": "Upgrade Azure network API version to 2018-07-01, so that EnableTcpReset could be enabled on Azure standard loadbalancer (SLB). ([#78012](https://github.com/kubernetes/kubernetes/pull/78012), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78012",
-    "pr_number": 78012,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.3"
-  },
-  "78016": {
-    "commit": "1fba88884be78abe1b231c1c72989b9588751a7a",
-    "text": "kubelet: fix fail to close kubelet-\u003eAPI connections on heartbeat failure when bootstrapping or client certificate rotation is disabled",
-    "markdown": "kubelet: fix fail to close kubelet-\u003eAPI connections on heartbeat failure when bootstrapping or client certificate rotation is disabled ([#78016](https://github.com/kubernetes/kubernetes/pull/78016), [@gaorong](https://github.com/gaorong))",
-    "author": "gaorong",
-    "author_url": "https://github.com/gaorong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78016",
-    "pr_number": 78016,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.3"
-  },
-  "78029": {
-    "commit": "0f8009b96d7dc3a7a9af25c0ce8d488f7a190b44",
-    "text": "Active watches of custom resources now terminate properly if the CRD is modified.",
-    "markdown": "Active watches of custom resources now terminate properly if the CRD is modified. ([#78029](https://github.com/kubernetes/kubernetes/pull/78029), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78029",
-    "pr_number": 78029,
-    "areas": ["apiserver", "custom-resources"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.3"
-  },
-  "78030": {
-    "commit": "648cb98a44fe8cd47d93bba153dce5b434e0df75",
-    "text": "CoreDNS is now version 1.5.0\n    - A `ready` plugin has been included to report pod readiness\n    - The `proxy` plugin has been deprecated. The `forward` plugin is to be used instead.\n    - CoreDNS fixes the logging now that kubernetes’ client lib switched to klog from glog.",
-    "markdown": "CoreDNS is now version 1.5.0\n      - A `ready` plugin has been included to report pod readiness\n      - The `proxy` plugin has been deprecated. The `forward` plugin is to be used instead.\n      - CoreDNS fixes the logging now that kubernetes’ client lib switched to klog from glog. ([#78030](https://github.com/kubernetes/kubernetes/pull/78030), [@rajansandeep](https://github.com/rajansandeep))",
-    "author": "rajansandeep",
-    "author_url": "https://github.com/rajansandeep",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78030",
-    "pr_number": 78030,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.3"
-  },
-  "78053": {
-    "commit": "2f7eaa1ee38b93491e15b98328d02f43b93c3eba",
-    "text": "kubeadm: implement CRI detection for Windows worker nodes",
-    "markdown": "kubeadm: implement CRI detection for Windows worker nodes ([#78053](https://github.com/kubernetes/kubernetes/pull/78053), [@ksubrmnn](https://github.com/ksubrmnn))",
-    "author": "ksubrmnn",
-    "author_url": "https://github.com/ksubrmnn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78053",
-    "pr_number": 78053,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.3"
-  },
-  "78059": {
-    "commit": "df4b3c866add70c4b7c1697c6139829eb3a6ccb8",
-    "text": "fix KUBE_SERVER_PLATFORMS null error when cross compiling kubectl for non-linux platform",
-    "markdown": "fix KUBE_SERVER_PLATFORMS null error when cross compiling kubectl for non-linux platform ([#78059](https://github.com/kubernetes/kubernetes/pull/78059), [@figo](https://github.com/figo))",
-    "author": "figo",
-    "author_url": "https://github.com/figo",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78059",
-    "pr_number": 78059,
-    "kinds": ["bug"],
-    "sigs": ["release"],
-    "release_version": "1.14.3"
-  },
-  "78061": {
-    "commit": "b3c1bbb4710f4b178250ae1f5a61be6508501f93",
-    "text": "Introducing new semantic for metric \"volume_operation_total_seconds\" to be the end to end latency of volume provisioning/deletion. Existing metric \"storage_operation_duration_seconds\" will remain untouched however exposed to the following potential issues:\n1. for volume's provisioned/deleted via external provisioner/deleter, \"storage_operation_duration_seconds\" will NOT wait for the external operation to be done before reporting latency metric (effectively close to 0). This will be fixed by using \"volume_operation_total_seconds\" instead\n2. if there's a transient error happened during \"provisioning/deletion\", i.e., a volume is still in-use while a deleteVolume has been called, original \"storage_operation_duration_seconds\" will NOT wait until a volume has been finally deleted before reporting a not accurate latency metric. The newly implemented metric \"volume_operation_total_seconds\", however, wait util a provisioning/deletion operation has been fully executed.\n\nPotential impacts:\nIf an SLO/alert has been defined based on \"volume_operation_total_seconds\", it might get violated because of the more accurate metric might be significantly larger than previously reported. The metric is defined to be a histogram and the new semantic could change the distribution.",
-    "markdown": "Introducing new semantic for metric \"volume_operation_total_seconds\" to be the end to end latency of volume provisioning/deletion. Existing metric \"storage_operation_duration_seconds\" will remain untouched however exposed to the following potential issues:\n  1. for volume's provisioned/deleted via external provisioner/deleter, \"storage_operation_duration_seconds\" will NOT wait for the external operation to be done before reporting latency metric (effectively close to 0). This will be fixed by using \"volume_operation_total_seconds\" instead\n  2. if there's a transient error happened during \"provisioning/deletion\", i.e., a volume is still in-use while a deleteVolume has been called, original \"storage_operation_duration_seconds\" will NOT wait until a volume has been finally deleted before reporting a not accurate latency metric. The newly implemented metric \"volume_operation_total_seconds\", however, wait util a provisioning/deletion operation has been fully executed.\n  \n  Potential impacts:\n  If an SLO/alert has been defined based on \"volume_operation_total_seconds\", it might get violated because of the more accurate metric might be significantly larger than previously reported. The metric is defined to be a histogram and the new semantic could change the distribution. ([#78061](https://github.com/kubernetes/kubernetes/pull/78061), [@yuxiangqian](https://github.com/yuxiangqian))\n\n  Courtesy of SIG Apps, and SIG Storage",
-    "author": "yuxiangqian",
-    "author_url": "https://github.com/yuxiangqian",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78061",
-    "pr_number": 78061,
-    "kinds": ["feature"],
-    "sigs": ["apps", "storage"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "78135": {
-    "commit": "cd9f430941fa1f27adb57027e0c220272580672b",
-    "text": "Admission webhooks can now register for a single version of a resource (for example, `apps/v1 deployments`) and be called when any other version of that resource is modified (for example `extensions/v1beta1 deployments`). This allows new versions of a resource to be handled by admission webhooks without needing to update every webhook to understand the new version. See the API documentation for the `matchPolicy: Equivalent` option in MutatingWebhookConfiguration and ValidatingWebhookConfiguration types.",
-    "markdown": "Admission webhooks can now register for a single version of a resource (for example, `apps/v1 deployments`) and be called when any other version of that resource is modified (for example `extensions/v1beta1 deployments`). This allows new versions of a resource to be handled by admission webhooks without needing to update every webhook to understand the new version. See the API documentation for the `matchPolicy: Equivalent` option in MutatingWebhookConfiguration and ValidatingWebhookConfiguration types. ([#78135](https://github.com/kubernetes/kubernetes/pull/78135), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery, and SIG Testing",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78135",
-    "pr_number": 78135,
-    "areas": ["apiserver", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "testing"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "78137": {
-    "commit": "5cb9405ad8617de7029f5d92fb14d9a1c2107494",
-    "text": "Fix vSphere SAML token auth when using Zones",
-    "markdown": "Fix vSphere SAML token auth when using Zones ([#78137](https://github.com/kubernetes/kubernetes/pull/78137), [@dougm](https://github.com/dougm))",
-    "author": "dougm",
-    "author_url": "https://github.com/dougm",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78137",
-    "pr_number": 78137,
-    "areas": ["cloudprovider", "dependency", "provider/vmware"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.3"
-  },
-  "78183": {
-    "commit": "5c314535d1944f1e1fa0fa272de31a1d28baab88",
-    "text": "Ensures that the node-role.kubernetes.io/master taint is applied to the master with NoSchedule on GCE.",
-    "markdown": "Ensures that the node-role.kubernetes.io/master taint is applied to the master with NoSchedule on GCE. ([#78183](https://github.com/kubernetes/kubernetes/pull/78183), [@cheftako](https://github.com/cheftako))",
-    "author": "cheftako",
-    "author_url": "https://github.com/cheftako",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78183",
-    "pr_number": 78183,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.3"
-  },
-  "78200": {
-    "commit": "a27d3aef304c5f11a5330bea150edf2d041c1c18",
-    "text": "reverts an aws-ebs volume provisioner optimization as we need to further discuss a viable optimization",
-    "markdown": "reverts an aws-ebs volume provisioner optimization as we need to further discuss a viable optimization ([#78200](https://github.com/kubernetes/kubernetes/pull/78200), [@zhan849](https://github.com/zhan849))",
-    "author": "zhan849",
-    "author_url": "https://github.com/zhan849",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78200",
-    "pr_number": 78200,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.3"
-  },
-  "78242": {
-    "commit": "bb7898ba19084e6f52b86aeb2f00cf6e2b71acba",
-    "text": "Azure cloud provider could now be configured by Kubernetes secrets and a new option `cloudConfigType` is introduced, whose candicate values are `file`, `secret` and `merge` (default is `merge`).\n\n\nSince Azure cloud provider would read Kubernetes secrets, the following RBAC should be configured:\n\n  apiVersion: rbac.authorization.k8s.io/v1beta1\n  kind: ClusterRole\n  metadata:\n    labels:\n      kubernetes.io/cluster-service: \"true\"\n    name: system:azure-cloud-provider-secret-getter\n  rules:\n  - apiGroups: [\"\"]\n    resources: [\"secrets\"]\n    resourceNames: [\"azure-cloud-provider\"]\n    verbs:\n    - get\n  ---\n  apiVersion: rbac.authorization.k8s.io/v1beta1\n  kind: ClusterRoleBinding\n  metadata:\n    labels:\n      kubernetes.io/cluster-service: \"true\"\n    name: system:azure-cloud-provider-secret-getter\n  roleRef:\n    apiGroup: rbac.authorization.k8s.io\n    kind: ClusterRole\n    name: system:azure-cloud-provider-secret-getter\n  subjects:\n  - kind: ServiceAccount\n    name: azure-cloud-provider\n    namespace: kube-system",
-    "markdown": "Azure cloud provider could now be configured by Kubernetes secrets and a new option `cloudConfigType` is introduced, whose candicate values are `file`, `secret` and `merge` (default is `merge`).\n  \n  \n  Since Azure cloud provider would read Kubernetes secrets, the following RBAC should be configured:\n  \n    apiVersion: rbac.authorization.k8s.io/v1beta1\n    kind: ClusterRole\n    metadata:\n      labels:\n        kubernetes.io/cluster-service: \"true\"\n      name: system:azure-cloud-provider-secret-getter\n    rules:\n    - apiGroups: [\"\"]\n      resources: [\"secrets\"]\n      resourceNames: [\"azure-cloud-provider\"]\n      verbs:\n      - get\n    ---\n    apiVersion: rbac.authorization.k8s.io/v1beta1\n    kind: ClusterRoleBinding\n    metadata:\n      labels:\n        kubernetes.io/cluster-service: \"true\"\n      name: system:azure-cloud-provider-secret-getter\n    roleRef:\n      apiGroup: rbac.authorization.k8s.io\n      kind: ClusterRole\n      name: system:azure-cloud-provider-secret-getter\n    subjects:\n    - kind: ServiceAccount\n      name: azure-cloud-provider\n      namespace: kube-system ([#78242](https://github.com/kubernetes/kubernetes/pull/78242), [@feiskyer](https://github.com/feiskyer))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78242",
-    "pr_number": 78242,
-    "areas": ["cloudprovider", "dependency"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "action_required": true,
-    "release_version": "1.14.3"
-  },
-  "78261": {
-    "commit": "78254d555a2958483780818ac83f899ff9aa6296",
-    "text": "Fix broken detection of non-root image user ID",
-    "markdown": "Fix broken detection of non-root image user ID ([#78261](https://github.com/kubernetes/kubernetes/pull/78261), [@tallclair](https://github.com/tallclair))",
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78261",
-    "pr_number": 78261,
-    "areas": ["kubelet", "security"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.3"
-  },
-  "78262": {
-    "commit": "bc32307314ab3fcb5ed30979872e2885c33c9e36",
-    "text": "Finalizer Protection for Service LoadBalancers is now added as Alpha (disabled by default). This feature ensures the Service resource is not fully deleted until the correlating load balancer resources are deleted.",
-    "markdown": "Finalizer Protection for Service LoadBalancers is now added as Alpha (disabled by default). This feature ensures the Service resource is not fully deleted until the correlating load balancer resources are deleted. ([#78262](https://github.com/kubernetes/kubernetes/pull/78262), [@MrHohn](https://github.com/MrHohn))\n\n  Courtesy of SIG Apps, SIG Cloud Provider, SIG Network, and SIG Testing",
-    "author": "MrHohn",
-    "author_url": "https://github.com/MrHohn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78262",
-    "pr_number": 78262,
-    "areas": ["cloudprovider", "test"],
-    "kinds": ["feature"],
-    "sigs": ["apps", "cloud-provider", "network", "testing"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "78272": {
-    "commit": "48848731605845c129945f1de146507ca2e5d8cb",
-    "text": "GCE: Disable the Windows defender to work around a bug that could cause nodes to crash and reboot",
-    "markdown": "GCE: Disable the Windows defender to work around a bug that could cause nodes to crash and reboot ([#78272](https://github.com/kubernetes/kubernetes/pull/78272), [@yujuhong](https://github.com/yujuhong))",
-    "author": "yujuhong",
-    "author_url": "https://github.com/yujuhong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78272",
-    "pr_number": 78272,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.3"
-  },
-  "78276": {
-    "commit": "f14b128ea6429a34b83578d4c2a69a99296a8d31",
-    "text": "Use zone from node for topology aware aws-ebs volume creation to reduce unnecessary cloud provider calls",
-    "markdown": "Use zone from node for topology aware aws-ebs volume creation to reduce unnecessary cloud provider calls ([#78276](https://github.com/kubernetes/kubernetes/pull/78276), [@zhan849](https://github.com/zhan849))",
-    "author": "zhan849",
-    "author_url": "https://github.com/zhan849",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78276",
-    "pr_number": 78276,
-    "kinds": ["bug"],
-    "sigs": ["scalability", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.3"
-  },
-  "78298": {
-    "commit": "fcc9f166fafde843ad9f9d02325bfcd782aae317",
-    "text": "fix azure retry issue when return 2XX with error",
-    "markdown": "fix azure retry issue when return 2XX with error ([#78298](https://github.com/kubernetes/kubernetes/pull/78298), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78298",
-    "pr_number": 78298,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.3"
-  },
-  "78313": {
-    "commit": "fa7c0e870f90324999f3ae55d5623b3267bcd68f",
-    "text": "Fixes CVE-2019-11248: /debug/pprof exposed on kubelet's healthz port",
-    "markdown": "Fixes CVE-2019-11248: /debug/pprof exposed on kubelet's healthz port ([#78313](https://github.com/kubernetes/kubernetes/pull/78313), [@tallclair](https://github.com/tallclair))",
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78313",
-    "pr_number": 78313,
-    "areas": ["apiserver", "kubelet", "security"],
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["api-machinery", "node"],
-    "duplicate": true,
-    "release_version": "1.14.3"
-  },
-  "78323": {
-    "commit": "eaece39c8692456e14dad5d908f6fb3c347f0ab3",
-    "text": "The dockershim container runtime now accepts the `docker` runtime handler from a RuntimeClass.",
-    "markdown": "The dockershim container runtime now accepts the `docker` runtime handler from a RuntimeClass. ([#78323](https://github.com/kubernetes/kubernetes/pull/78323), [@tallclair](https://github.com/tallclair))\n\n  Courtesy of SIG Node, and SIG Testing",
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78323",
-    "pr_number": 78323,
-    "areas": ["kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["node", "testing"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "78325": {
-    "commit": "d817a7e9a0e449c997c743526db43c9d718c6364",
-    "text": "The storageVersionHash feature is beta now. \"StorageVersionHash\" is a field in the discovery document of each resource. It allows clients to detect if the storage version of that resource has changed. Its value must be treated as opaque by clients. Only equality comparison on the value is valid.",
-    "markdown": "The storageVersionHash feature is beta now. \"StorageVersionHash\" is a field in the discovery document of each resource. It allows clients to detect if the storage version of that resource has changed. Its value must be treated as opaque by clients. Only equality comparison on the value is valid. ([#78325](https://github.com/kubernetes/kubernetes/pull/78325), [@caesarxuchao](https://github.com/caesarxuchao))\n\n  Courtesy of SIG API Machinery, and SIG Testing",
-    "author": "caesarxuchao",
-    "author_url": "https://github.com/caesarxuchao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78325",
-    "pr_number": 78325,
-    "areas": ["apiserver", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "testing"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "78330": {
-    "commit": "8d5052eddf07bc0f55c00d00af0f034589d9d988",
-    "text": "Add support for Azure Disk plugin to csi-translation-lib",
-    "markdown": "Add support for Azure Disk plugin to csi-translation-lib ([#78330](https://github.com/kubernetes/kubernetes/pull/78330), [@andyzhangx](https://github.com/andyzhangx))\n\n  Courtesy of SIG Storage",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78330",
-    "pr_number": 78330,
-    "kinds": ["bug", "feature"],
-    "sigs": ["storage"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "78344": {
-    "commit": "9139adea23e5b268da87f9fc2c32de64c32df682",
-    "text": "Remove deprecated flag --cleanup-iptables from kube-proxy",
-    "markdown": "Remove deprecated flag --cleanup-iptables from kube-proxy ([#78344](https://github.com/kubernetes/kubernetes/pull/78344), [@aramase](https://github.com/aramase))",
-    "author": "aramase",
-    "author_url": "https://github.com/aramase",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78344",
-    "pr_number": 78344,
-    "kinds": ["cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.14.3"
-  },
-  "78356": {
-    "commit": "ade661c7cb684aab4fd18db4928e95ca8ca42338",
-    "text": "Add support for Azure File plugin to csi-translation-lib",
-    "markdown": "Add support for Azure File plugin to csi-translation-lib ([#78356](https://github.com/kubernetes/kubernetes/pull/78356), [@andyzhangx](https://github.com/andyzhangx))\n\n  Courtesy of SIG Storage",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78356",
-    "pr_number": 78356,
-    "kinds": ["feature"],
-    "sigs": ["storage"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "78385": {
-    "commit": "3ccb63bb1b4d8a9e9a5e32bdd2750e44a6067ad4",
-    "text": "Allow trailing dots in the externalName of Services of type ExternalName.",
-    "markdown": "Allow trailing dots in the externalName of Services of type ExternalName. ([#78385](https://github.com/kubernetes/kubernetes/pull/78385), [@thz](https://github.com/thz))",
-    "author": "thz",
-    "author_url": "https://github.com/thz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78385",
-    "pr_number": 78385,
-    "kinds": ["bug"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.14.3"
-  },
-  "78395": {
-    "commit": "2fb7b6074a05c082b9bbf0429b47bbfc096cbb1b",
-    "text": "Increase log level to 2 for IPVS graceful termination",
-    "markdown": "Increase log level to 2 for IPVS graceful termination ([#78395](https://github.com/kubernetes/kubernetes/pull/78395), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78395",
-    "pr_number": 78395,
-    "areas": ["ipvs"],
-    "kinds": ["cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.14.3"
-  },
-  "78399": {
-    "commit": "cc30c0d9a496dfa19919536fe2950df1cf8dd98f",
-    "text": "ACTION REQUIRED  The deprecated flag --conntrack-max has been removed from kube-proxy. Users of this flag should switch to --conntrack-min and --conntrack-max-per-core instead.",
-    "markdown": "ACTION REQUIRED  The deprecated flag --conntrack-max has been removed from kube-proxy. Users of this flag should switch to --conntrack-min and --conntrack-max-per-core instead. ([#78399](https://github.com/kubernetes/kubernetes/pull/78399), [@rikatz](https://github.com/rikatz))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Network",
-    "author": "rikatz",
-    "author_url": "https://github.com/rikatz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78399",
-    "pr_number": 78399,
-    "areas": ["kubeadm"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "action_required": true,
-    "release_version": "1.14.3"
-  },
-  "78408": {
-    "commit": "de817520826acd44b6098d2a95841850418befc6",
-    "text": "kubeadm: a new command `kubeadm upgrade node` is introduced for upgrading nodes (both secondary control-plane nodes and worker nodes) \nThe command `kubeadm upgrade node config` is now deprecated; use `kubeadm upgrade node` instead.\nThe command `kubeadm upgrade node experimental-control-plane` is now deprecated; use `kubeadm upgrade node` instead.",
-    "markdown": "kubeadm: a new command `kubeadm upgrade node` is introduced for upgrading nodes (both secondary control-plane nodes and worker nodes) \n  The command `kubeadm upgrade node config` is now deprecated; use `kubeadm upgrade node` instead.\n  The command `kubeadm upgrade node experimental-control-plane` is now deprecated; use `kubeadm upgrade node` instead. ([#78408](https://github.com/kubernetes/kubernetes/pull/78408), [@fabriziopandini](https://github.com/fabriziopandini))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78408",
-    "pr_number": 78408,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "78426": {
-    "commit": "b18846cf4cdc1a96d47842bc0ae37d88e7342345",
-    "text": "CustomResourcesDefinition conversion via webhooks is promoted to beta. It requires that spec.preserveUnknownFields is set to false.",
-    "markdown": "CustomResourcesDefinition conversion via webhooks is promoted to beta. It requires that spec.preserveUnknownFields is set to false. ([#78426](https://github.com/kubernetes/kubernetes/pull/78426), [@sttts](https://github.com/sttts))\n\n  Courtesy of SIG API Machinery, and SIG Testing",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78426",
-    "pr_number": 78426,
-    "areas": ["custom-resources", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "testing"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "78428": {
-    "commit": "9b14c223758084ddd1c020992d508be9a7db4596",
-    "text": "Fixed panic in kube-proxy when parsing iptables-save output",
-    "markdown": "Fixed panic in kube-proxy when parsing iptables-save output ([#78428](https://github.com/kubernetes/kubernetes/pull/78428), [@luksa](https://github.com/luksa))",
-    "author": "luksa",
-    "author_url": "https://github.com/luksa",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78428",
-    "pr_number": 78428,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.3"
-  },
-  "78445": {
-    "commit": "38afd9815e6c9dbfe3db77ead724befb05f3c1e4",
-    "text": "`kubectl scale job`, deprecated since 1.10, has been removed",
-    "markdown": "`kubectl scale job`, deprecated since 1.10, has been removed ([#78445](https://github.com/kubernetes/kubernetes/pull/78445), [@soltysh](https://github.com/soltysh))",
-    "author": "soltysh",
-    "author_url": "https://github.com/soltysh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78445",
-    "pr_number": 78445,
-    "areas": ["kubectl", "test"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.3"
-  },
-  "78448": {
-    "commit": "225859eb3ee896959e98199003f2ec08eac1fec4",
-    "text": "Promote resource quota scope selector to GA",
-    "markdown": "Promote resource quota scope selector to GA ([#78448](https://github.com/kubernetes/kubernetes/pull/78448), [@ravisantoshgudimetla](https://github.com/ravisantoshgudimetla))",
-    "author": "ravisantoshgudimetla",
-    "author_url": "https://github.com/ravisantoshgudimetla",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78448",
-    "pr_number": 78448,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.3"
-  },
-  "78452": {
-    "commit": "44a3bf47550e129cfb4fa13b3bca2c819ca936cd",
-    "text": "kubeadm: flag “--experimental-control-plane” is now deprecated. use “--control-plane” instead\nkubeadm: flag “--experimental-upload-certs” is now deprecated. use “--upload-certs” instead",
-    "markdown": "kubeadm: flag “--experimental-control-plane” is now deprecated. use “--control-plane” instead\n  kubeadm: flag “--experimental-upload-certs” is now deprecated. use “--upload-certs” instead ([#78452](https://github.com/kubernetes/kubernetes/pull/78452), [@fabriziopandini](https://github.com/fabriziopandini))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78452",
-    "pr_number": 78452,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "78453": {
-    "commit": "15a01d5fa58266ee8957a48f8ea72ea435890b05",
-    "text": "CustomResourceDefinition with invalid regular expression in the pattern field of OpenAPI v3 validation schemas are not considere structural.",
-    "markdown": "CustomResourceDefinition with invalid regular expression in the pattern field of OpenAPI v3 validation schemas are not considere structural. ([#78453](https://github.com/kubernetes/kubernetes/pull/78453), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78453",
-    "pr_number": 78453,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.3"
-  },
-  "78475": {
-    "commit": "8362e9c4f9646aaf045a4d213835610f25348f76",
-    "text": "Till this release, iscsi plugin was waiting 10 seconds for a path to appear in the device list. However this timeout is not enough or less than default device discovery timeout in most of the systems which cause certain device to be not accounted for the volume. This timeout has been lifted to 30seconds from this release and it should help to avoid mount issues due to device discovery.",
-    "markdown": "Till this release, iscsi plugin was waiting 10 seconds for a path to appear in the device list. However this timeout is not enough or less than default device discovery timeout in most of the systems which cause certain device to be not accounted for the volume. This timeout has been lifted to 30seconds from this release and it should help to avoid mount issues due to device discovery. ([#78475](https://github.com/kubernetes/kubernetes/pull/78475), [@humblec](https://github.com/humblec))",
-    "author": "humblec",
-    "author_url": "https://github.com/humblec",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78475",
-    "pr_number": 78475,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.3"
-  },
-  "78497": {
-    "commit": "a7a28979b1d0b7740c0bc02fef7e25ef937fdc41",
-    "text": "Supports configure accessLogs for AWS NLB",
-    "markdown": "Supports configure accessLogs for AWS NLB ([#78497](https://github.com/kubernetes/kubernetes/pull/78497), [@M00nF1sh](https://github.com/M00nF1sh))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "M00nF1sh",
-    "author_url": "https://github.com/M00nF1sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78497",
-    "pr_number": 78497,
-    "areas": ["cloudprovider"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.14.3"
-  },
-  "78503": {
-    "commit": "9f85c5c4b56940f88cb854615efad4e9f0c66f20",
-    "text": "Horizontal Pod Autoscaling can now scale targets up even when one or more metrics are invalid/unavailable as long as one metric indicates a scale up should occur.",
-    "markdown": "Horizontal Pod Autoscaling can now scale targets up even when one or more metrics are invalid/unavailable as long as one metric indicates a scale up should occur. ([#78503](https://github.com/kubernetes/kubernetes/pull/78503), [@gjtempleton](https://github.com/gjtempleton))",
-    "author": "gjtempleton",
-    "author_url": "https://github.com/gjtempleton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78503",
-    "pr_number": 78503,
-    "kinds": ["cleanup"],
-    "sigs": ["apps", "autoscaling"],
-    "duplicate": true,
-    "release_version": "1.14.3"
-  },
-  "78504": {
-    "commit": "6b6bdc760a6a23060ce20f9458e93646fe8c038e",
-    "text": "Deprecate kubelet cAdvisor json endpoints",
-    "markdown": "Deprecate kubelet cAdvisor json endpoints ([#78504](https://github.com/kubernetes/kubernetes/pull/78504), [@dashpole](https://github.com/dashpole))",
-    "author": "dashpole",
-    "author_url": "https://github.com/dashpole",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78504",
-    "pr_number": 78504,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.14.3"
-  },
-  "78505": {
-    "commit": "bada1c6b1eef959825c3dca1d3944e1ac4c31184",
-    "text": "Added objectSelector to admission webhook configurations. objectSelector is evaluated the oldObject and newObject that would be sent to the webhook, and is considered to match if either object matches the selector. A null object (oldObject in the case of create, or newObject in the case of delete) or an object that cannot have labels (like a DeploymentRollback or a PodProxyOptions object) is not considered to match. Use the object selector only if the webhook is opt-in, because end users may skip the admission webhook by setting the labels.",
-    "markdown": "Added objectSelector to admission webhook configurations. objectSelector is evaluated the oldObject and newObject that would be sent to the webhook, and is considered to match if either object matches the selector. A null object (oldObject in the case of create, or newObject in the case of delete) or an object that cannot have labels (like a DeploymentRollback or a PodProxyOptions object) is not considered to match. Use the object selector only if the webhook is opt-in, because end users may skip the admission webhook by setting the labels. ([#78505](https://github.com/kubernetes/kubernetes/pull/78505), [@caesarxuchao](https://github.com/caesarxuchao))",
-    "author": "caesarxuchao",
-    "author_url": "https://github.com/caesarxuchao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78505",
-    "pr_number": 78505,
-    "areas": ["admission-control", "apiserver", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.3"
-  },
-  "78507": {
-    "commit": "21c9c7a6ab4786fadb4056f0a49f1969acc340db",
-    "text": "Windows nodes on GCE now have the Windows firewall enabled by default.",
-    "markdown": "Windows nodes on GCE now have the Windows firewall enabled by default. ([#78507](https://github.com/kubernetes/kubernetes/pull/78507), [@pjh](https://github.com/pjh))",
-    "author": "pjh",
-    "author_url": "https://github.com/pjh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78507",
-    "pr_number": 78507,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.3"
-  },
-  "78528": {
-    "commit": "d910055d35314f877786074b00b678e6387172ff",
-    "text": "Windows Kubelet nodes will now correctly search the default location for Docker credentials (`%USERPROFILE%\\.docker\\config.json`) when pulling images from a private registry (https://kubernetes.io/docs/concepts/containers/images/\u0026#35;configuring-nodes-to-authenticate-to-a-private-registry)",
-    "markdown": "Windows Kubelet nodes will now correctly search the default location for Docker credentials (`%USERPROFILE%\\.docker\\config.json`) when pulling images from a private registry (https://kubernetes.io/docs/concepts/containers/images/\u0026#35;configuring-nodes-to-authenticate-to-a-private-registry) ([#78528](https://github.com/kubernetes/kubernetes/pull/78528), [@claudiubelu](https://github.com/claudiubelu))",
-    "author": "claudiubelu",
-    "author_url": "https://github.com/claudiubelu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78528",
-    "pr_number": 78528,
-    "kinds": ["bug"],
-    "sigs": ["windows"],
-    "release_version": "1.14.3"
-  },
-  "78542": {
-    "commit": "c81ae93159c9d02577bc8037a1e416dc166db9f6",
-    "text": "kubeadm: the mixture of \"--config\" and \"--certificate-key\" is no longer allowed. The InitConfiguration and JoinConfiguration objects now support the \"certificateKey\" field and this field should be used instead of the command line argument in case a configuration file is already passed.",
-    "markdown": "kubeadm: the mixture of \"--config\" and \"--certificate-key\" is no longer allowed. The InitConfiguration and JoinConfiguration objects now support the \"certificateKey\" field and this field should be used instead of the command line argument in case a configuration file is already passed. ([#78542](https://github.com/kubernetes/kubernetes/pull/78542), [@neolit123](https://github.com/neolit123))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78542",
-    "pr_number": 78542,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "action_required": true,
-    "release_version": "1.14.3"
-  },
-  "78545": {
-    "commit": "16aeec248538144517b6463c4414646f4b2eeeaf",
-    "text": "kubeadm: revert the CoreDNS version to 1.3.1",
-    "markdown": "kubeadm: revert the CoreDNS version to 1.3.1 ([#78545](https://github.com/kubernetes/kubernetes/pull/78545), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78545",
-    "pr_number": 78545,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.3"
-  },
-  "78581": {
-    "commit": "ac458e9d89df895424586f8c135b86c4ee07dc91",
-    "text": "Route controller now respects rate limiting to the cloud provider on deletion, previously it was only for create.",
-    "markdown": "Route controller now respects rate limiting to the cloud provider on deletion, previously it was only for create. ([#78581](https://github.com/kubernetes/kubernetes/pull/78581), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78581",
-    "pr_number": 78581,
-    "kinds": ["bug"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.14.3"
-  },
-  "78594": {
-    "commit": "acb321e614d21dd7cb8ea55265a9f55363140c39",
-    "text": "Fixes a memory leak in Kubelet on Windows caused by not not closing containers when fetching container metrics",
-    "markdown": "Fixes a memory leak in Kubelet on Windows caused by not not closing containers when fetching container metrics ([#78594](https://github.com/kubernetes/kubernetes/pull/78594), [@benmoss](https://github.com/benmoss))",
-    "author": "benmoss",
-    "author_url": "https://github.com/benmoss",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78594",
-    "pr_number": 78594,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.3"
-  },
-  "78611": {
-    "commit": "9c807d483e4321101a203a5b25b35d5d61d69a55",
-    "text": "Fix error handling for loading initCfg in kubeadm upgrade and apply",
-    "markdown": "Fix error handling for loading initCfg in kubeadm upgrade and apply ([#78611](https://github.com/kubernetes/kubernetes/pull/78611), [@odinuge](https://github.com/odinuge))",
-    "author": "odinuge",
-    "author_url": "https://github.com/odinuge",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78611",
-    "pr_number": 78611,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.3"
-  },
-  "78612": {
-    "commit": "f532d5ca4b792948b9471afb8373ed098eb1a5ad",
-    "text": "Windows kube-proxy will wait for HNS network creation on start",
-    "markdown": "Windows kube-proxy will wait for HNS network creation on start ([#78612](https://github.com/kubernetes/kubernetes/pull/78612), [@ksubrmnn](https://github.com/ksubrmnn))",
-    "author": "ksubrmnn",
-    "author_url": "https://github.com/ksubrmnn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78612",
-    "pr_number": 78612,
-    "kinds": ["bug"],
-    "sigs": ["network", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.3"
-  }
-}
diff --git a/src/assets/release-notes-1.14.4.json b/src/assets/release-notes-1.14.4.json
deleted file mode 100644
index 8d023ff4..00000000
--- a/src/assets/release-notes-1.14.4.json
+++ /dev/null
@@ -1,1073 +0,0 @@
-{
-  "68108": {
-    "commit": "3d4124f2e083591c98e8a874760011781d3ee15c",
-    "text": "Minor fix ConstructVolumeSpec in iSCSI plugin",
-    "markdown": "Minor fix ConstructVolumeSpec in iSCSI plugin ([#68108](https://github.com/kubernetes/kubernetes/pull/68108), [@wenjun93](https://github.com/wenjun93))",
-    "author": "wenjun93",
-    "author_url": "https://github.com/wenjun93",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/68108",
-    "pr_number": 68108,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.4"
-  },
-  "69263": {
-    "commit": "bf55a99eb861262b59375eb3a11d13b39db80031",
-    "text": "Creates an annotation 'service.beta.kubernetes.io/aws-load-balancer-eip-allocations' to assign AWS EIP to the newly created Network Load Balancer. Number of allocations and subnets must match.",
-    "markdown": "Creates an annotation 'service.beta.kubernetes.io/aws-load-balancer-eip-allocations' to assign AWS EIP to the newly created Network Load Balancer. Number of allocations and subnets must match. ([#69263](https://github.com/kubernetes/kubernetes/pull/69263), [@brooksgarrett](https://github.com/brooksgarrett))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "brooksgarrett",
-    "author_url": "https://github.com/brooksgarrett",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/69263",
-    "pr_number": 69263,
-    "areas": ["cloudprovider"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.14.4"
-  },
-  "70672": {
-    "commit": "86e5099b0760a7ef4faf431f1b605bd99177f384",
-    "text": "The following APIs are no longer served by default:\n- All resources under `apps/v1beta1` and `apps/v1beta2` - use `apps/v1` instead\n- `daemonsets`, `deployments`, `replicasets` resources under `extensions/v1beta1` - use `apps/v1` instead\n- `networkpolicies` resources under `extensions/v1beta1` - use `networking.k8s.io/v1` instead\n- `podsecuritypolicies` resources under `extensions/v1beta1` - use `policy/v1beta1` instead\n\nServing these resources can be temporarily re-enabled using the `--runtime-config` apiserver flag. \n- `apps/v1beta1=true`\n- `apps/v1beta2=true`\n- `extensions/v1beta1/daemonsets=true,extensions/v1beta1/deployments=true,extensions/v1beta1/replicasets=true,extensions/v1beta1/networkpolicies=true,extensions/v1beta1/podsecuritypolicies=true`\n\nThe ability to serve these resources will be completely removed in v1.18.",
-    "markdown": "The following APIs are no longer served by default:\n  - All resources under `apps/v1beta1` and `apps/v1beta2` - use `apps/v1` instead\n  - `daemonsets`, `deployments`, `replicasets` resources under `extensions/v1beta1` - use `apps/v1` instead\n  - `networkpolicies` resources under `extensions/v1beta1` - use `networking.k8s.io/v1` instead\n  - `podsecuritypolicies` resources under `extensions/v1beta1` - use `policy/v1beta1` instead\n  \n  Serving these resources can be temporarily re-enabled using the `--runtime-config` apiserver flag. \n  - `apps/v1beta1=true`\n  - `apps/v1beta2=true`\n  - `extensions/v1beta1/daemonsets=true,extensions/v1beta1/deployments=true,extensions/v1beta1/replicasets=true,extensions/v1beta1/networkpolicies=true,extensions/v1beta1/podsecuritypolicies=true`\n  \n  The ability to serve these resources will be completely removed in v1.18. ([#70672](https://github.com/kubernetes/kubernetes/pull/70672), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Network, and SIG Testing",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/70672",
-    "pr_number": 70672,
-    "areas": ["apiserver", "test"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["api-machinery", "apps", "network", "testing"],
-    "action_required": true,
-    "release_version": "1.14.4"
-  },
-  "72178": {
-    "commit": "c6bc1639c3f99e0bbfbf45f3b7fd2da3a9971ac5",
-    "text": "Remove deprecated cloud providers - ovirt, cloudstack and photon",
-    "markdown": "Remove deprecated cloud providers - ovirt, cloudstack and photon ([#72178](https://github.com/kubernetes/kubernetes/pull/72178), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72178",
-    "pr_number": 72178,
-    "areas": ["cloudprovider", "code-organization", "dependency", "kubeadm", "kubelet", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cloud-provider", "cluster-lifecycle", "node", "storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "72420": {
-    "commit": "2a82853614b710d30c334d41723661beb1ffcfd3",
-    "text": "The deprecated node condition type 'OutOfDisk' has been removed. Use the 'DiskPressure' condition instead.",
-    "markdown": "The deprecated node condition type 'OutOfDisk' has been removed. Use the 'DiskPressure' condition instead. ([#72420](https://github.com/kubernetes/kubernetes/pull/72420), [@Pingan2017](https://github.com/Pingan2017))",
-    "author": "Pingan2017",
-    "author_url": "https://github.com/Pingan2017",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72420",
-    "pr_number": 72420,
-    "areas": ["kubelet"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["api-machinery", "apps", "architecture", "node", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "73977": {
-    "commit": "a33840e0232aaf2b5c14c1c754d9189d7e841ec1",
-    "text": "To configure controller manager to use ipv6dual stack:\nuse --cluster-cidr=\"\u003ccidr1\u003e,\u003ccidr2\u003e\".\n\nNotes:\n \n1. Only the first two cidrs are used (soft limits for Alpha, might be lifted later on). \n2. Only the \"RangeAllocator\" (default) is allowed as a value for --cidr-allocator-type . Cloud allocators are not compatible with ipv6dualstack",
-    "markdown": "To configure controller manager to use ipv6dual stack:\n  use --cluster-cidr=\"\u003ccidr1\u003e,\u003ccidr2\u003e\".\n  \n  Notes:\n   \n  1. Only the first two cidrs are used (soft limits for Alpha, might be lifted later on). \n  2. Only the \"RangeAllocator\" (default) is allowed as a value for --cidr-allocator-type . Cloud allocators are not compatible with ipv6dualstack ([#73977](https://github.com/kubernetes/kubernetes/pull/73977), [@khenidak](https://github.com/khenidak))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Architecture, SIG CLI, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Network, SIG Node, SIG Release, SIG Scheduling, and SIG Testing",
-    "author": "khenidak",
-    "author_url": "https://github.com/khenidak",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73977",
-    "pr_number": 73977,
-    "areas": [
-      "apiserver",
-      "cloudprovider",
-      "code-generation",
-      "dependency",
-      "kubectl",
-      "kubelet",
-      "release-eng",
-      "test"
-    ],
-    "kinds": ["api-change", "feature"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "architecture",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "network",
-      "node",
-      "release",
-      "scheduling",
-      "testing"
-    ],
-    "feature": true,
-    "release_version": "1.14.4"
-  },
-  "73982": {
-    "commit": "75a39b70072954634169f7bc0731795019d04bac",
-    "text": "Fixes invalid \"time stamp is the future\" error when kubectl cp-ing a file",
-    "markdown": "Fixes invalid \"time stamp is the future\" error when kubectl cp-ing a file ([#73982](https://github.com/kubernetes/kubernetes/pull/73982), [@tanshanshan](https://github.com/tanshanshan))",
-    "author": "tanshanshan",
-    "author_url": "https://github.com/tanshanshan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73982",
-    "pr_number": 73982,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.4"
-  },
-  "74943": {
-    "commit": "bf86300241e51f011040d208e43f1cd2a5c0378a",
-    "text": "kubectl logs --selector will support --tail=-1.",
-    "markdown": "kubectl logs --selector will support --tail=-1. ([#74943](https://github.com/kubernetes/kubernetes/pull/74943), [@JishanXing](https://github.com/JishanXing))",
-    "author": "JishanXing",
-    "author_url": "https://github.com/JishanXing",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74943",
-    "pr_number": 74943,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "cli"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "75223": {
-    "commit": "cff6e420056f65469a30460ecc45111c3b71a799",
-    "text": "Init container resource requests now impact pod QoS class",
-    "markdown": "Init container resource requests now impact pod QoS class ([#75223](https://github.com/kubernetes/kubernetes/pull/75223), [@sjenning](https://github.com/sjenning))",
-    "author": "sjenning",
-    "author_url": "https://github.com/sjenning",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75223",
-    "pr_number": 75223,
-    "kinds": ["bug"],
-    "sigs": ["apps", "node"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "75234": {
-    "commit": "3a598d7f92675c09ee2ea5bb76470fc27f65ab29",
-    "text": "fix pod stuck issue due to corrupt mnt point in flexvol plugin, call Unmount if PathExists returns any error",
-    "markdown": "fix pod stuck issue due to corrupt mnt point in flexvol plugin, call Unmount if PathExists returns any error ([#75234](https://github.com/kubernetes/kubernetes/pull/75234), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75234",
-    "pr_number": 75234,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "76124": {
-    "commit": "1a80962db6fc7138f4d2925a82d7e74cd5ec4fcd",
-    "text": "Fix CRD validation error on 'items' field.",
-    "markdown": "Fix CRD validation error on 'items' field. ([#76124](https://github.com/kubernetes/kubernetes/pull/76124), [@tossmilestone](https://github.com/tossmilestone))",
-    "author": "tossmilestone",
-    "author_url": "https://github.com/tossmilestone",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76124",
-    "pr_number": 76124,
-    "areas": [
-      "apiserver",
-      "cloudprovider",
-      "code-generation",
-      "custom-resources",
-      "dependency",
-      "kubectl"
-    ],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "node"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "76968": {
-    "commit": "155f4a649b39bbc5627993537fc280ae43c13a75",
-    "text": "Addition of Overhead field to the PodSpec and RuntimeClass types as part of the Pod Overhead KEP",
-    "markdown": "Addition of Overhead field to the PodSpec and RuntimeClass types as part of the Pod Overhead KEP ([#76968](https://github.com/kubernetes/kubernetes/pull/76968), [@egernst](https://github.com/egernst))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG CLI, and SIG Node",
-    "author": "egernst",
-    "author_url": "https://github.com/egernst",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76968",
-    "pr_number": 76968,
-    "areas": ["kubectl", "kubelet"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "cli", "node"],
-    "feature": true,
-    "release_version": "1.14.4"
-  },
-  "77595": {
-    "commit": "22fb6fd174660e5e0e46b9af41482efcfb057f7e",
-    "text": "Integrated volume limits for in-tree and CSI volumes into one scheduler predicate.",
-    "markdown": "Integrated volume limits for in-tree and CSI volumes into one scheduler predicate. ([#77595](https://github.com/kubernetes/kubernetes/pull/77595), [@bertinatto](https://github.com/bertinatto))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Instrumentation, SIG Node, SIG Release, SIG Scheduling, SIG Storage, and SIG Testing",
-    "author": "bertinatto",
-    "author_url": "https://github.com/bertinatto",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77595",
-    "pr_number": 77595,
-    "areas": ["cloudprovider", "e2e-test-framework", "kubeadm", "kubelet", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "auth",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "node",
-      "release",
-      "scheduling",
-      "storage",
-      "testing"
-    ],
-    "feature": true,
-    "release_version": "1.14.4"
-  },
-  "77712": {
-    "commit": "4b8a1163aa5193e32164dab267d867ab4d085ee1",
-    "text": "if targetPort is changed that will process by service controller",
-    "markdown": "if targetPort is changed that will process by service controller ([#77712](https://github.com/kubernetes/kubernetes/pull/77712), [@Sn0rt](https://github.com/Sn0rt))\n\n  Courtesy of SIG Apps, and SIG Network",
-    "author": "Sn0rt",
-    "author_url": "https://github.com/Sn0rt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77712",
-    "pr_number": 77712,
-    "kinds": ["feature"],
-    "sigs": ["apps", "network"],
-    "feature": true,
-    "release_version": "1.14.4"
-  },
-  "77792": {
-    "commit": "003c4e51555f6e9fce8b62ba6916d5b70b2f3010",
-    "text": "kubeadm: prevent PSP blocking of upgrade image prepull by using a non-root user",
-    "markdown": "kubeadm: prevent PSP blocking of upgrade image prepull by using a non-root user ([#77792](https://github.com/kubernetes/kubernetes/pull/77792), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77792",
-    "pr_number": 77792,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.4"
-  },
-  "77819": {
-    "commit": "024c7bd84f1e9a386778e14068f57c3b1e193565",
-    "text": "A new client `k8s.io/client-go/metadata.Client` has been added for accessing objects generically. This client makes it easier to retrieve only the metadata (the `metadata` sub-section) from resources on the cluster in an efficient manner for use cases that deal with objects generically, like the garbage collector, quota, or the namespace controller. The client asks the server to return a `meta.k8s.io/v1 PartialObjectMetadata` object for list, get, delete, watch, and patch operations on both normal APIs and custom resources which can be encoded in protobuf for additional work. If the server does not yet support this API the client will gracefully fall back to JSON and transform the response objects into PartialObjectMetadata.",
-    "markdown": "A new client `k8s.io/client-go/metadata.Client` has been added for accessing objects generically. This client makes it easier to retrieve only the metadata (the `metadata` sub-section) from resources on the cluster in an efficient manner for use cases that deal with objects generically, like the garbage collector, quota, or the namespace controller. The client asks the server to return a `meta.k8s.io/v1 PartialObjectMetadata` object for list, get, delete, watch, and patch operations on both normal APIs and custom resources which can be encoded in protobuf for additional work. If the server does not yet support this API the client will gracefully fall back to JSON and transform the response objects into PartialObjectMetadata. ([#77819](https://github.com/kubernetes/kubernetes/pull/77819), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77819",
-    "pr_number": 77819,
-    "areas": ["apiserver", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "78005": {
-    "commit": "d8695d06b7191db56ebbbc0340da263833c9bb6f",
-    "text": "Add Pre-filter extension point to the scheduling framework.",
-    "markdown": "Add Pre-filter extension point to the scheduling framework. ([#78005](https://github.com/kubernetes/kubernetes/pull/78005), [@ahg-g](https://github.com/ahg-g))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "ahg-g",
-    "author_url": "https://github.com/ahg-g",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78005",
-    "pr_number": 78005,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.4"
-  },
-  "78294": {
-    "commit": "eee3e976d871bcbf97821a0fa9c129a4c568ac5e",
-    "text": "Removed deprecated flag `--resource-container` from kube-proxy.\n\nThe deprecated `--resource-container` flag has been removed from kube-proxy, and specifying it will now cause an error.  The behavior is now as if you specified `--resource-container=\"\"`.  If you previously specified a non-empty `--resource-container`, you can no longer do so as of kubernetes 1.16.",
-    "markdown": "Removed deprecated flag `--resource-container` from kube-proxy.\n  \n  The deprecated `--resource-container` flag has been removed from kube-proxy, and specifying it will now cause an error.  The behavior is now as if you specified `--resource-container=\"\"`.  If you previously specified a non-empty `--resource-container`, you can no longer do so as of kubernetes 1.16. ([#78294](https://github.com/kubernetes/kubernetes/pull/78294), [@vllry](https://github.com/vllry))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Network",
-    "author": "vllry",
-    "author_url": "https://github.com/vllry",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78294",
-    "pr_number": 78294,
-    "areas": ["kubeadm"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "action_required": true,
-    "release_version": "1.14.4"
-  },
-  "78458": {
-    "commit": "ae3c44d043769305c6add665cd81c16319479ba7",
-    "text": "The kube-apiserver has improved behavior for both startup and shutdown sequences and also now exposes `\\readyz` for readiness checking. Readyz includes all existing healthz checks but also adds a shutdown check. When a cluster admin initiates a shutdown, the kube-apiserver will try to process existing requests (for the duration of request timeout) before killing the apiserver process.  \n\nThe apiserver also now takes an optional flag \"--maximum-startup-sequence-duration\". This allows you to explicitly define an upper bound on the apiserver startup sequences before healthz begins to fail. By keeping the kubelet liveness initial delay short, this can enable quick kubelet recovery as soon as we have a boot sequence which has not completed in our expected time frame, despite lack of completion from longer boot sequences (like RBAC). Kube-apiserver behavior when the value of this flag is zero is backwards compatible (this is as the defaulted value of the flag).",
-    "markdown": "The kube-apiserver has improved behavior for both startup and shutdown sequences and also now exposes `\\readyz` for readiness checking. Readyz includes all existing healthz checks but also adds a shutdown check. When a cluster admin initiates a shutdown, the kube-apiserver will try to process existing requests (for the duration of request timeout) before killing the apiserver process.  \n  \n  The apiserver also now takes an optional flag \"--maximum-startup-sequence-duration\". This allows you to explicitly define an upper bound on the apiserver startup sequences before healthz begins to fail. By keeping the kubelet liveness initial delay short, this can enable quick kubelet recovery as soon as we have a boot sequence which has not completed in our expected time frame, despite lack of completion from longer boot sequences (like RBAC). Kube-apiserver behavior when the value of this flag is zero is backwards compatible (this is as the defaulted value of the flag). ([#78458](https://github.com/kubernetes/kubernetes/pull/78458), [@logicalhan](https://github.com/logicalhan))",
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78458",
-    "pr_number": 78458,
-    "areas": ["apiserver", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "auth", "cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "78484": {
-    "commit": "e4f15883522673a57349641f7601a816470a98b6",
-    "text": "Introduce a new admission controller for RuntimeClass. Initially, RuntimeClass will be used to apply the pod overhead associated with a given RuntimeClass to the Pod.Spec if a corresponding RuntimeClassName is specified.\n\nPodOverhead is an alpha feature as of Kubernetes 1.16.",
-    "markdown": "Introduce a new admission controller for RuntimeClass. Initially, RuntimeClass will be used to apply the pod overhead associated with a given RuntimeClass to the Pod.Spec if a corresponding RuntimeClassName is specified.\n  \n  PodOverhead is an alpha feature as of Kubernetes 1.16. ([#78484](https://github.com/kubernetes/kubernetes/pull/78484), [@egernst](https://github.com/egernst))\n\n  Courtesy of SIG API Machinery, and SIG Node",
-    "author": "egernst",
-    "author_url": "https://github.com/egernst",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78484",
-    "pr_number": 78484,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "node"],
-    "feature": true,
-    "release_version": "1.14.4"
-  },
-  "78495": {
-    "commit": "ca6113fb8bc0752aaaa2956b7d900059acf138f5",
-    "text": "Fix kubelet errors in AArch64 with huge page sizes smaller than 1MiB",
-    "markdown": "Fix kubelet errors in AArch64 with huge page sizes smaller than 1MiB ([#78495](https://github.com/kubernetes/kubernetes/pull/78495), [@odinuge](https://github.com/odinuge))",
-    "author": "odinuge",
-    "author_url": "https://github.com/odinuge",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78495",
-    "pr_number": 78495,
-    "areas": ["apiserver", "dependency", "kubectl", "kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli", "node", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "78513": {
-    "commit": "6efc5a8c9aa9d0c6509b59cb5e2761a56cf4f860",
-    "text": "Add Bind extension point of the scheduling framework",
-    "markdown": "Add Bind extension point of the scheduling framework ([#78513](https://github.com/kubernetes/kubernetes/pull/78513), [@chenchun](https://github.com/chenchun))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "chenchun",
-    "author_url": "https://github.com/chenchun",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78513",
-    "pr_number": 78513,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.4"
-  },
-  "78522": {
-    "commit": "304a2b191d550cea615687b68c5ff0da4ca5bd2c",
-    "text": "StorageOS volumes now show correct mount information (node and mount time) in the StorageOS administration CLI and UI.",
-    "markdown": "StorageOS volumes now show correct mount information (node and mount time) in the StorageOS administration CLI and UI. ([#78522](https://github.com/kubernetes/kubernetes/pull/78522), [@croomes](https://github.com/croomes))",
-    "author": "croomes",
-    "author_url": "https://github.com/croomes",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78522",
-    "pr_number": 78522,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.4"
-  },
-  "78540": {
-    "commit": "a7cde2e70fc2c82ada19367b65a27e9a67173038",
-    "text": "KMS Providers will install a healthz check for the status of kms-pluign in kube-apiservers' encryption config.",
-    "markdown": "KMS Providers will install a healthz check for the status of kms-pluign in kube-apiservers' encryption config. ([#78540](https://github.com/kubernetes/kubernetes/pull/78540), [@immutableT](https://github.com/immutableT))\n\n  Courtesy of SIG API Machinery, SIG Auth, SIG CLI, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Node, and SIG Testing",
-    "author": "immutableT",
-    "author_url": "https://github.com/immutableT",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78540",
-    "pr_number": 78540,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": [
-      "api-machinery",
-      "auth",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "node",
-      "testing"
-    ],
-    "feature": true,
-    "release_version": "1.14.4"
-  },
-  "78556": {
-    "commit": "fadb63c7eb9e3c081d5f3e73fd54c70e0cf47fca",
-    "text": "kubeadm: introduce deterministic ordering for the certificates generation in the phase command \"kubeadm init phase certs\" .",
-    "markdown": "kubeadm: introduce deterministic ordering for the certificates generation in the phase command \"kubeadm init phase certs\" . ([#78556](https://github.com/kubernetes/kubernetes/pull/78556), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78556",
-    "pr_number": 78556,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.4"
-  },
-  "78595": {
-    "commit": "333081e79c79b779063935fdbec00452c7511f2b",
-    "text": "Fix issue with kubelet waiting on invalid devicepath on AWS",
-    "markdown": "Fix issue with kubelet waiting on invalid devicepath on AWS ([#78595](https://github.com/kubernetes/kubernetes/pull/78595), [@gnufied](https://github.com/gnufied))",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78595",
-    "pr_number": 78595,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.4"
-  },
-  "78608": {
-    "commit": "ca12f110470c6c649202f5229daeb044143f9451",
-    "text": "Fix admission metrics histogram bucket sizes to cover 25ms to ~2.5 seconds.",
-    "markdown": "Fix admission metrics histogram bucket sizes to cover 25ms to ~2.5 seconds. ([#78608](https://github.com/kubernetes/kubernetes/pull/78608), [@jpbetz](https://github.com/jpbetz))",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78608",
-    "pr_number": 78608,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.4"
-  },
-  "78625": {
-    "commit": "54f7e5269c9641fc436fbd622865975d7d90e4a0",
-    "text": "kubeadm: use the service-cidr flag to pass the desired service CIDR to the kube-controller-manager via its service-cluster-ip-range flag.",
-    "markdown": "kubeadm: use the service-cidr flag to pass the desired service CIDR to the kube-controller-manager via its service-cluster-ip-range flag. ([#78625](https://github.com/kubernetes/kubernetes/pull/78625), [@Arvinderpal](https://github.com/Arvinderpal))",
-    "author": "Arvinderpal",
-    "author_url": "https://github.com/Arvinderpal",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78625",
-    "pr_number": 78625,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "78668": {
-    "commit": "3cd41a5a0bc4bbdf156bf925316713a1c41d634a",
-    "text": "kube-up.sh scripts now disable the KubeletPodResources feature for Windows nodes, due to issue \u0026#35;78628.",
-    "markdown": "kube-up.sh scripts now disable the KubeletPodResources feature for Windows nodes, due to issue \u0026#35;78628. ([#78668](https://github.com/kubernetes/kubernetes/pull/78668), [@mtaufen](https://github.com/mtaufen))",
-    "author": "mtaufen",
-    "author_url": "https://github.com/mtaufen",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78668",
-    "pr_number": 78668,
-    "kinds": ["failing-test"],
-    "sigs": ["cluster-lifecycle", "node", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "78672": {
-    "commit": "a03bc34313c4353b79e70807942edb7c68e3a8fd",
-    "text": "Volume expansion is enabled in the default GCE storageclass",
-    "markdown": "Volume expansion is enabled in the default GCE storageclass ([#78672](https://github.com/kubernetes/kubernetes/pull/78672), [@msau42](https://github.com/msau42))",
-    "author": "msau42",
-    "author_url": "https://github.com/msau42",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78672",
-    "pr_number": 78672,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.4"
-  },
-  "78687": {
-    "commit": "0e499be526bc91ad2d2e187428da12daa03a1eae",
-    "text": "fix pod list return value of framework.WaitForPodsWithLabelRunningReady",
-    "markdown": "fix pod list return value of framework.WaitForPodsWithLabelRunningReady ([#78687](https://github.com/kubernetes/kubernetes/pull/78687), [@pohly](https://github.com/pohly))",
-    "author": "pohly",
-    "author_url": "https://github.com/pohly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78687",
-    "pr_number": 78687,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["bug"],
-    "sigs": ["testing"],
-    "release_version": "1.14.4"
-  },
-  "78691": {
-    "commit": "32ec6c212ec9415f604ffc1f4c1f29b782968ff1",
-    "text": "Revert the CoreDNS version to 1.3.1",
-    "markdown": "Revert the CoreDNS version to 1.3.1 ([#78691](https://github.com/kubernetes/kubernetes/pull/78691), [@rajansandeep](https://github.com/rajansandeep))",
-    "author": "rajansandeep",
-    "author_url": "https://github.com/rajansandeep",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78691",
-    "pr_number": 78691,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "78696": {
-    "commit": "e168e558f46159e202ca34b6cb7a243110cc41b9",
-    "text": "Revert Promotion of resource quota scope selector to GA",
-    "markdown": "Revert Promotion of resource quota scope selector to GA ([#78696](https://github.com/kubernetes/kubernetes/pull/78696), [@ravisantoshgudimetla](https://github.com/ravisantoshgudimetla))",
-    "author": "ravisantoshgudimetla",
-    "author_url": "https://github.com/ravisantoshgudimetla",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78696",
-    "pr_number": 78696,
-    "kinds": ["cleanup"],
-    "sigs": ["node", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "78700": {
-    "commit": "59594523c80160e181ce49bd399b59868e1f6cbc",
-    "text": "fix: retry detach azure disk issue\ntry to only update vm if detach a non-existing disk when got \u003c200, error\u003e after detach disk operation",
-    "markdown": "fix: retry detach azure disk issue\n  try to only update vm if detach a non-existing disk when got \u003c200, error\u003e after detach disk operation ([#78700](https://github.com/kubernetes/kubernetes/pull/78700), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78700",
-    "pr_number": 78700,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.4"
-  },
-  "78713": {
-    "commit": "f3f2b4066b154575823ad8582e47f81e3db847cd",
-    "text": "Fixed a spurious error where update requests to the status subresource of multi-version custom resources would complain about an incorrect API version.",
-    "markdown": "Fixed a spurious error where update requests to the status subresource of multi-version custom resources would complain about an incorrect API version. ([#78713](https://github.com/kubernetes/kubernetes/pull/78713), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78713",
-    "pr_number": 78713,
-    "areas": ["apiserver", "custom-resources"],
-    "kinds": ["api-change", "bug", "flake"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.4"
-  },
-  "78815": {
-    "commit": "5fe910191a6a1a64872e249c4c431b0169d763ed",
-    "text": "CRDs get support for x-kuberntes-int-or-string to allow faithful representation of IntOrString types in CustomResources.",
-    "markdown": "CRDs get support for x-kuberntes-int-or-string to allow faithful representation of IntOrString types in CustomResources. ([#78815](https://github.com/kubernetes/kubernetes/pull/78815), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78815",
-    "pr_number": 78815,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.4"
-  },
-  "78866": {
-    "commit": "d873167e8f816f5f83846bc4587c3047452293bd",
-    "text": "Update Cluster Autoscaler to 1.15.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.15.0",
-    "markdown": "Update Cluster Autoscaler to 1.15.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.15.0 ([#78866](https://github.com/kubernetes/kubernetes/pull/78866), [@losipiuk](https://github.com/losipiuk))",
-    "author": "losipiuk",
-    "author_url": "https://github.com/losipiuk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78866",
-    "pr_number": 78866,
-    "kinds": ["bug"],
-    "sigs": ["autoscaling", "cluster-lifecycle"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "78868": {
-    "commit": "f4a62ad660944e13f51b95cedfe15742580472fb",
-    "text": "Mounts /home/kubernetes/bin/nvidia/vulkan/icd.d on the host to /etc/vulkan/icd.d inside containers requesting GPU.",
-    "markdown": "Mounts /home/kubernetes/bin/nvidia/vulkan/icd.d on the host to /etc/vulkan/icd.d inside containers requesting GPU. ([#78868](https://github.com/kubernetes/kubernetes/pull/78868), [@chardch](https://github.com/chardch))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "chardch",
-    "author_url": "https://github.com/chardch",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78868",
-    "pr_number": 78868,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.4"
-  },
-  "78876": {
-    "commit": "8de1569ddae62e8fab559fe6bd210a5d6100a277",
-    "text": "vSphere: allow SAML token delegation (required for Zones support)",
-    "markdown": "vSphere: allow SAML token delegation (required for Zones support) ([#78876](https://github.com/kubernetes/kubernetes/pull/78876), [@dougm](https://github.com/dougm))",
-    "author": "dougm",
-    "author_url": "https://github.com/dougm",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78876",
-    "pr_number": 78876,
-    "areas": ["cloudprovider", "provider/vmware"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.4"
-  },
-  "78885": {
-    "commit": "0a9373b89ad88a6e72be5208e949d50c1216d95a",
-    "text": "Resolves spurious rollouts of workload controllers when upgrading the API server, due to incorrect defaulting of an alpha procMount field in pods",
-    "markdown": "Resolves spurious rollouts of workload controllers when upgrading the API server, due to incorrect defaulting of an alpha procMount field in pods ([#78885](https://github.com/kubernetes/kubernetes/pull/78885), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78885",
-    "pr_number": 78885,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "78908": {
-    "commit": "6f73ab22197ad0753a01f76f6f86caa03a51f548",
-    "text": "kubelet now accepts a --cni-cache-dir option, which defaults to /var/lib/cni/cache, where CNI stores cache files.",
-    "markdown": "kubelet now accepts a --cni-cache-dir option, which defaults to /var/lib/cni/cache, where CNI stores cache files. ([#78908](https://github.com/kubernetes/kubernetes/pull/78908), [@dcbw](https://github.com/dcbw))",
-    "author": "dcbw",
-    "author_url": "https://github.com/dcbw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78908",
-    "pr_number": 78908,
-    "areas": ["dependency", "kubeadm", "kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "network", "node", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "78915": {
-    "commit": "4f29960cb2058c4f556bf1e5ba49c9a29a501e70",
-    "text": "kubeadm: implement retry logic for certain ConfigMap failures when joining nodes",
-    "markdown": "kubeadm: implement retry logic for certain ConfigMap failures when joining nodes ([#78915](https://github.com/kubernetes/kubernetes/pull/78915), [@ereslibre](https://github.com/ereslibre))",
-    "author": "ereslibre",
-    "author_url": "https://github.com/ereslibre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78915",
-    "pr_number": 78915,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.4"
-  },
-  "78958": {
-    "commit": "bd35a48815a46191e47f2e139f0fb90e4cc8eb15",
-    "text": "update to use go 1.12.6",
-    "markdown": "update to use go 1.12.6 ([#78958](https://github.com/kubernetes/kubernetes/pull/78958), [@tao12345666333](https://github.com/tao12345666333))\n\n  Courtesy of SIG Testing",
-    "author": "tao12345666333",
-    "author_url": "https://github.com/tao12345666333",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78958",
-    "pr_number": 78958,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["testing"],
-    "feature": true,
-    "release_version": "1.14.4"
-  },
-  "78974": {
-    "commit": "1782653fe87245fdbd49f96dbeee3bb4db13fa2e",
-    "text": "fix a bug that pods not be deleted from unmatched nodes by daemon controller",
-    "markdown": "fix a bug that pods not be deleted from unmatched nodes by daemon controller ([#78974](https://github.com/kubernetes/kubernetes/pull/78974), [@DaiHao](https://github.com/DaiHao))",
-    "author": "DaiHao",
-    "author_url": "https://github.com/DaiHao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78974",
-    "pr_number": 78974,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.14.4"
-  },
-  "78991": {
-    "commit": "1a15ce14473fb60bb817d1f29147ae7b70a0987e",
-    "text": "Reflector watchHandler Warning log 'The resourceVersion for the provided watch is too old.' is now logged as Info.",
-    "markdown": "Reflector watchHandler Warning log 'The resourceVersion for the provided watch is too old.' is now logged as Info. ([#78991](https://github.com/kubernetes/kubernetes/pull/78991), [@sallyom](https://github.com/sallyom))",
-    "author": "sallyom",
-    "author_url": "https://github.com/sallyom",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78991",
-    "pr_number": 78991,
-    "areas": ["apiserver", "cloudprovider", "dependency"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "78999": {
-    "commit": "101f9ff7031f02aec7c2b6fdb9001a7fc0b630a7",
-    "text": "Fix a string comparison bug in IPVS graceful termination where UDP real servers are not deleted.",
-    "markdown": "Fix a string comparison bug in IPVS graceful termination where UDP real servers are not deleted. ([#78999](https://github.com/kubernetes/kubernetes/pull/78999), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78999",
-    "pr_number": 78999,
-    "areas": ["ipvs"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.4"
-  },
-  "79000": {
-    "commit": "af9d54db76624c04e014c0d6d2146f83781d4617",
-    "text": "Fixes a bug where `kubectl set config` hangs and uses 100% CPU on some invalid property names",
-    "markdown": "Fixes a bug where `kubectl set config` hangs and uses 100% CPU on some invalid property names ([#79000](https://github.com/kubernetes/kubernetes/pull/79000), [@pswica](https://github.com/pswica))",
-    "author": "pswica",
-    "author_url": "https://github.com/pswica",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79000",
-    "pr_number": 79000,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.4"
-  },
-  "79002": {
-    "commit": "6602f6159660b8fa17c743246f37acfeb3baee5e",
-    "text": "Enable cadvisor ProcessMetrics collecting.",
-    "markdown": "Enable cadvisor ProcessMetrics collecting. ([#79002](https://github.com/kubernetes/kubernetes/pull/79002), [@jiayingz](https://github.com/jiayingz))\n\n  Courtesy of SIG Node",
-    "author": "jiayingz",
-    "author_url": "https://github.com/jiayingz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79002",
-    "pr_number": 79002,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.14.4"
-  },
-  "79020": {
-    "commit": "9120d427dab4a69ddcf22597f81b574fb1af96bc",
-    "text": "fix: make azure disk URI as case insensitive",
-    "markdown": "fix: make azure disk URI as case insensitive ([#79020](https://github.com/kubernetes/kubernetes/pull/79020), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79020",
-    "pr_number": 79020,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.4"
-  },
-  "79073": {
-    "commit": "caf6b67ce2a2d2cd18b9634c51198275ffe638ad",
-    "text": "Remove pids cgroup controller requirement when related feature gates are disabled",
-    "markdown": "Remove pids cgroup controller requirement when related feature gates are disabled ([#79073](https://github.com/kubernetes/kubernetes/pull/79073), [@rafatio](https://github.com/rafatio))",
-    "author": "rafatio",
-    "author_url": "https://github.com/rafatio",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79073",
-    "pr_number": 79073,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.4"
-  },
-  "79094": {
-    "commit": "efa043151c458bc36c28931f8a69693b6d454bb7",
-    "text": "fix kubelet fail to delete orphaned pod directory when the kubelet's pods directory (default is \"/var/lib/kubelet/pods\") symbolically links to another disk device's directory",
-    "markdown": "fix kubelet fail to delete orphaned pod directory when the kubelet's pods directory (default is \"/var/lib/kubelet/pods\") symbolically links to another disk device's directory ([#79094](https://github.com/kubernetes/kubernetes/pull/79094), [@gaorong](https://github.com/gaorong))",
-    "author": "gaorong",
-    "author_url": "https://github.com/gaorong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79094",
-    "pr_number": 79094,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.4"
-  },
-  "79106": {
-    "commit": "c8b45cd25c18e65798dde49fc7011495ea6021d5",
-    "text": "The behavior of the default handler for 404 requests fro the GCE Ingress load balancer is slightly modified in the sense that it now exports metrics using prometheus. The metrics exported include: \n- http_404_request_total  (the number of 404 requests handled)\n- http_404_request_duration_ms (the amount of time the server took to respond in ms)\n\nAlso includes percentile groupings. The directory for the default 404 handler includes instructions on how to enable prometheus for monitoring and setting alerts.",
-    "markdown": "The behavior of the default handler for 404 requests fro the GCE Ingress load balancer is slightly modified in the sense that it now exports metrics using prometheus. The metrics exported include: \n  - http_404_request_total  (the number of 404 requests handled)\n  - http_404_request_duration_ms (the amount of time the server took to respond in ms)\n  \n  Also includes percentile groupings. The directory for the default 404 handler includes instructions on how to enable prometheus for monitoring and setting alerts. ([#79106](https://github.com/kubernetes/kubernetes/pull/79106), [@vbannai](https://github.com/vbannai))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "vbannai",
-    "author_url": "https://github.com/vbannai",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79106",
-    "pr_number": 79106,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.4"
-  },
-  "79114": {
-    "commit": "5cd9faf74a401df6845e34c857accfc5b2aa74c1",
-    "text": "The CRD handler now properly re-creates stale CR storage to reflect CRD update.",
-    "markdown": "The CRD handler now properly re-creates stale CR storage to reflect CRD update. ([#79114](https://github.com/kubernetes/kubernetes/pull/79114), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79114",
-    "pr_number": 79114,
-    "areas": ["custom-resources"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.4"
-  },
-  "79165": {
-    "commit": "4683054ff6b476b8ebaac25460e76c65f528e1a0",
-    "text": "kubeadm: fix a potential panic if kubeadm discovers an invalid, existing kubeconfig file",
-    "markdown": "kubeadm: fix a potential panic if kubeadm discovers an invalid, existing kubeconfig file ([#79165](https://github.com/kubernetes/kubernetes/pull/79165), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79165",
-    "pr_number": 79165,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.4"
-  },
-  "79262": {
-    "commit": "8ce45b642ef6cdc45ca73301effd7837b663e7f3",
-    "text": "The GA PodPriority feature gate is now on by default and cannot be disabled. The feature gate will be removed in v1.18.",
-    "markdown": "The GA PodPriority feature gate is now on by default and cannot be disabled. The feature gate will be removed in v1.18. ([#79262](https://github.com/kubernetes/kubernetes/pull/79262), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79262",
-    "pr_number": 79262,
-    "areas": ["apiserver", "kubelet", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "apps", "node", "scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "79270": {
-    "commit": "4c9fab5c4066e129fbd9b1ca505ef8ea440a2e50",
-    "text": "kubeadm: provide \"--control-plane-endpoint\" flag for `controlPlaneEndpoint`",
-    "markdown": "kubeadm: provide \"--control-plane-endpoint\" flag for `controlPlaneEndpoint` ([#79270](https://github.com/kubernetes/kubernetes/pull/79270), [@SataQiu](https://github.com/SataQiu))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79270",
-    "pr_number": 79270,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.4"
-  },
-  "79284": {
-    "commit": "e79dcc2174a996ddf85efecddb4f119e61b4d741",
-    "text": "When using the conformance test image, a new environment variable E2E_USE_GO_RUNNER will cause the tests to be run with the new Golang-based test runner rather than the current bash wrapper.",
-    "markdown": "When using the conformance test image, a new environment variable E2E_USE_GO_RUNNER will cause the tests to be run with the new Golang-based test runner rather than the current bash wrapper. ([#79284](https://github.com/kubernetes/kubernetes/pull/79284), [@johnSchnake](https://github.com/johnSchnake))\n\n  Courtesy of SIG Cluster Lifecycle, SIG Release, and SIG Testing",
-    "author": "johnSchnake",
-    "author_url": "https://github.com/johnSchnake",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79284",
-    "pr_number": 79284,
-    "areas": ["conformance"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "release", "testing"],
-    "feature": true,
-    "release_version": "1.14.4"
-  },
-  "79305": {
-    "commit": "ed9f340addb4679565ccca27f438a6324e225d2a",
-    "text": "- Node labels `beta.kubernetes.io/metadata-proxy-ready`, `beta.kubernetes.io/metadata-proxy-ready` and `beta.kubernetes.io/kube-proxy-ds-ready` are no longer added on new nodes.\n- ip-mask-agent addon starts to use the label `node.kubernetes.io/masq-agent-ds-ready` instead of `beta.kubernetes.io/masq-agent-ds-ready` as its node selector.\n- kube-proxy addon starts to use the label `node.kubernetes.io/kube-proxy-ds-ready` instead of `beta.kubernetes.io/kube-proxy-ds-ready` as its node selector.\n- metadata-proxy addon starts to use the label `cloud.google.com/metadata-proxy-ready` instead of `beta.kubernetes.io/metadata-proxy-ready` as its node selector.\n- Kubelet removes the ability to set `kubernetes.io` or `k8s.io` labels via --node-labels other than the [specifically allowed labels/prefixes](https://github.com/kubernetes/enhancements/blob/master/keps/sig-auth/0000-20170814-bounding-self-labeling-kubelets.md\u0026#35;proposal).",
-    "markdown": "- Node labels `beta.kubernetes.io/metadata-proxy-ready`, `beta.kubernetes.io/metadata-proxy-ready` and `beta.kubernetes.io/kube-proxy-ds-ready` are no longer added on new nodes.\n  - ip-mask-agent addon starts to use the label `node.kubernetes.io/masq-agent-ds-ready` instead of `beta.kubernetes.io/masq-agent-ds-ready` as its node selector.\n  - kube-proxy addon starts to use the label `node.kubernetes.io/kube-proxy-ds-ready` instead of `beta.kubernetes.io/kube-proxy-ds-ready` as its node selector.\n  - metadata-proxy addon starts to use the label `cloud.google.com/metadata-proxy-ready` instead of `beta.kubernetes.io/metadata-proxy-ready` as its node selector.\n  - Kubelet removes the ability to set `kubernetes.io` or `k8s.io` labels via --node-labels other than the [specifically allowed labels/prefixes](https://github.com/kubernetes/enhancements/blob/master/keps/sig-auth/0000-20170814-bounding-self-labeling-kubelets.md\u0026#35;proposal). ([#79305](https://github.com/kubernetes/kubernetes/pull/79305), [@paivagustavo](https://github.com/paivagustavo))\n\n  Courtesy of SIG Auth, SIG Cluster Lifecycle, and SIG Node",
-    "author": "paivagustavo",
-    "author_url": "https://github.com/paivagustavo",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79305",
-    "pr_number": 79305,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["auth", "cluster-lifecycle", "node"],
-    "action_required": true,
-    "release_version": "1.14.4"
-  },
-  "79307": {
-    "commit": "46469ee9d51d8e5760674bfca6ba2e374a9ab511",
-    "text": "Remove HugePages, VolumeScheduling, CustomPodDNS and PodReadinessGates feature flags",
-    "markdown": "Remove HugePages, VolumeScheduling, CustomPodDNS and PodReadinessGates feature flags ([#79307](https://github.com/kubernetes/kubernetes/pull/79307), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79307",
-    "pr_number": 79307,
-    "kinds": ["cleanup"],
-    "sigs": ["release"],
-    "release_version": "1.14.4"
-  },
-  "79309": {
-    "commit": "ad095324bfa99018e35fbac6d80c418d55c81064",
-    "text": "Remove CSIPersistentVolume feature gates",
-    "markdown": "Remove CSIPersistentVolume feature gates ([#79309](https://github.com/kubernetes/kubernetes/pull/79309), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79309",
-    "pr_number": 79309,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "apps", "auth", "node", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "79310": {
-    "commit": "2109c1a7a300e0ff41c7c0acd2dea00ba172c4b8",
-    "text": "Remove KubeletPluginsWatcher feature gates",
-    "markdown": "Remove KubeletPluginsWatcher feature gates ([#79310](https://github.com/kubernetes/kubernetes/pull/79310), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79310",
-    "pr_number": 79310,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["auth", "node", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "79326": {
-    "commit": "80a3177f17d9dc55f62cbf170c864f18b58c4fec",
-    "text": "Fix remove the etcd member from the cluster during a kubeadm reset.",
-    "markdown": "Fix remove the etcd member from the cluster during a kubeadm reset. ([#79326](https://github.com/kubernetes/kubernetes/pull/79326), [@bradbeam](https://github.com/bradbeam))",
-    "author": "bradbeam",
-    "author_url": "https://github.com/bradbeam",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79326",
-    "pr_number": 79326,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.4"
-  },
-  "79345": {
-    "commit": "47459465254b840b16220a163540631e416b22c2",
-    "text": "Fixes output of `kubectl get --watch-only` when watching a single resource",
-    "markdown": "Fixes output of `kubectl get --watch-only` when watching a single resource ([#79345](https://github.com/kubernetes/kubernetes/pull/79345), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79345",
-    "pr_number": 79345,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.4"
-  },
-  "79346": {
-    "commit": "5267a3dc730ff606cb5b1d1a4ee05ef04ae7ed3b",
-    "text": "Correct the maximum allowed insecure bind port for the kube-scheduler and kube-apiserver to 65535.",
-    "markdown": "Correct the maximum allowed insecure bind port for the kube-scheduler and kube-apiserver to 65535. ([#79346](https://github.com/kubernetes/kubernetes/pull/79346), [@ncdc](https://github.com/ncdc))",
-    "author": "ncdc",
-    "author_url": "https://github.com/ncdc",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79346",
-    "pr_number": 79346,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "79349": {
-    "commit": "c1fc717ac666dabbe7b723b0388fb04e5aa7e0ca",
-    "text": "Fix a bug in server printer that could cause kube-apiserver to panic.",
-    "markdown": "Fix a bug in server printer that could cause kube-apiserver to panic. ([#79349](https://github.com/kubernetes/kubernetes/pull/79349), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79349",
-    "pr_number": 79349,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.4"
-  },
-  "79375": {
-    "commit": "46667d0bbbcee0d8435feed3fe2fcb8c8977f9ab",
-    "text": "RateLimiter add a context-aware method, fix client-go request goruntine backlog in async timeout scene.",
-    "markdown": "RateLimiter add a context-aware method, fix client-go request goruntine backlog in async timeout scene. ([#79375](https://github.com/kubernetes/kubernetes/pull/79375), [@answer1991](https://github.com/answer1991))\n\n  Courtesy of SIG API Machinery",
-    "author": "answer1991",
-    "author_url": "https://github.com/answer1991",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79375",
-    "pr_number": 79375,
-    "kinds": ["bug", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.4"
-  },
-  "79391": {
-    "commit": "c8cee54240a0f36264b4de62423617dbc30b57aa",
-    "text": "Kubelet should now more reliably report the same primary node IP even if the set of node IPs reported by the CloudProvider changes.",
-    "markdown": "Kubelet should now more reliably report the same primary node IP even if the set of node IPs reported by the CloudProvider changes. ([#79391](https://github.com/kubernetes/kubernetes/pull/79391), [@danwinship](https://github.com/danwinship))",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79391",
-    "pr_number": 79391,
-    "areas": ["kubelet"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.4"
-  },
-  "79451": {
-    "commit": "b51f62186a75650f0021da9e3715c90609984cfc",
-    "text": "Fix a bug where kubelet would not retry pod sandbox creation when the restart policy of the pod is Never",
-    "markdown": "Fix a bug where kubelet would not retry pod sandbox creation when the restart policy of the pod is Never ([#79451](https://github.com/kubernetes/kubernetes/pull/79451), [@yujuhong](https://github.com/yujuhong))",
-    "author": "yujuhong",
-    "author_url": "https://github.com/yujuhong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79451",
-    "pr_number": 79451,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.4"
-  },
-  "79493": {
-    "commit": "dd8a000a77341a11b1616b3ead341184764652f3",
-    "text": "Fix duplicate error messages in cli commands",
-    "markdown": "Fix duplicate error messages in cli commands ([#79493](https://github.com/kubernetes/kubernetes/pull/79493), [@odinuge](https://github.com/odinuge))",
-    "author": "odinuge",
-    "author_url": "https://github.com/odinuge",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79493",
-    "pr_number": 79493,
-    "areas": ["apiserver", "kubeadm", "kubectl", "kubelet"],
-    "kinds": ["bug"],
-    "sigs": [
-      "api-machinery",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "network",
-      "node",
-      "release",
-      "scheduling"
-    ],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  },
-  "79498": {
-    "commit": "7db459a11bac8c092b0f319e4c5d6adaf2fe1912",
-    "text": "kubeadm: fix the bug that \"--cri-socket\" flag does not work for `kubeadm reset`",
-    "markdown": "kubeadm: fix the bug that \"--cri-socket\" flag does not work for `kubeadm reset` ([#79498](https://github.com/kubernetes/kubernetes/pull/79498), [@SataQiu](https://github.com/SataQiu))",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79498",
-    "pr_number": 79498,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.4"
-  },
-  "79504": {
-    "commit": "665e76d97623447d13bb3b33b68591a985b49c9d",
-    "text": "The alpha `metadata.initializers` field, deprecated in 1.13, has been removed.",
-    "markdown": "The alpha `metadata.initializers` field, deprecated in 1.13, has been removed. ([#79504](https://github.com/kubernetes/kubernetes/pull/79504), [@yue9944882](https://github.com/yue9944882))",
-    "author": "yue9944882",
-    "author_url": "https://github.com/yue9944882",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79504",
-    "pr_number": 79504,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.4"
-  },
-  "79514": {
-    "commit": "5019bb57a3e146bf7c19dfe178cbc34b456388dc",
-    "text": "Default resourceGroup should be used when the value of annotation azure-load-balancer-resource-group is an empty string.",
-    "markdown": "Default resourceGroup should be used when the value of annotation azure-load-balancer-resource-group is an empty string. ([#79514](https://github.com/kubernetes/kubernetes/pull/79514), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79514",
-    "pr_number": 79514,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.4"
-  },
-  "79529": {
-    "commit": "a3be4b6817ea7a62482196fe78f07e4fe7b5fde2",
-    "text": "changes timeout value in csi plugin from 15s to 2min which fixes the timeout issue",
-    "markdown": "changes timeout value in csi plugin from 15s to 2min which fixes the timeout issue ([#79529](https://github.com/kubernetes/kubernetes/pull/79529), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79529",
-    "pr_number": 79529,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.4"
-  },
-  "79534": {
-    "commit": "a5e3d7469149c704e32a9fd858be2a25c4ca9a2c",
-    "text": "Fix possible fd leak and closing of dirs in doSafeMakeDir",
-    "markdown": "Fix possible fd leak and closing of dirs in doSafeMakeDir ([#79534](https://github.com/kubernetes/kubernetes/pull/79534), [@odinuge](https://github.com/odinuge))",
-    "author": "odinuge",
-    "author_url": "https://github.com/odinuge",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79534",
-    "pr_number": 79534,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.4"
-  },
-  "79583": {
-    "commit": "38e8b8a9f65e66b779a0985e17868b036c1d601b",
-    "text": "Update Azure API versions (containerregistry --\u003e 2018-09-01, network --\u003e 2018-08-01)",
-    "markdown": "Update Azure API versions (containerregistry --\u003e 2018-09-01, network --\u003e 2018-08-01) ([#79583](https://github.com/kubernetes/kubernetes/pull/79583), [@justaugustus](https://github.com/justaugustus))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "justaugustus",
-    "author_url": "https://github.com/justaugustus",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79583",
-    "pr_number": 79583,
-    "areas": ["cloudprovider"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.14.4"
-  },
-  "79636": {
-    "commit": "8765ed365f9e206fd1296aaa216e3c9346671e40",
-    "text": "Fixes a bug in openapi published for custom resources using x-kubernetes-preserve-unknown-fields extensions, so that kubectl will allow sending unknown fields for that portion of the object.",
-    "markdown": "Fixes a bug in openapi published for custom resources using x-kubernetes-preserve-unknown-fields extensions, so that kubectl will allow sending unknown fields for that portion of the object. ([#79636](https://github.com/kubernetes/kubernetes/pull/79636), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79636",
-    "pr_number": 79636,
-    "areas": ["custom-resources", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.4"
-  }
-}
diff --git a/src/assets/release-notes-1.14.5.json b/src/assets/release-notes-1.14.5.json
deleted file mode 100644
index 06097c28..00000000
--- a/src/assets/release-notes-1.14.5.json
+++ /dev/null
@@ -1,1168 +0,0 @@
-{
-  "72416": {
-    "commit": "5625eadd2a3b73fc6bd328bc10ec238c7ab84e0f",
-    "text": "`kubectl get -w` now takes an `--output-watch-events` flag to indicate the event type (ADDED, MODIFIED, DELETED)",
-    "markdown": "`kubectl get -w` now takes an `--output-watch-events` flag to indicate the event type (ADDED, MODIFIED, DELETED) ([#72416](https://github.com/kubernetes/kubernetes/pull/72416), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG CLI, and SIG Testing",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72416",
-    "pr_number": 72416,
-    "areas": ["kubectl", "test"],
-    "kinds": ["feature"],
-    "sigs": ["cli", "testing"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "73923": {
-    "commit": "98c68df6069dc8f81c1f983609be60b2045861cb",
-    "text": "On Windows systems, %USERPROFILE% is now preferred over %HOMEDRIVE%\\%HOMEPATH% as the home folder if %HOMEDRIVE%\\%HOMEPATH% does not contain a .kube\\config file, and %USERPROFILE% exists and is writeable.",
-    "markdown": "On Windows systems, %USERPROFILE% is now preferred over %HOMEDRIVE%\\%HOMEPATH% as the home folder if %HOMEDRIVE%\\%HOMEPATH% does not contain a .kube\\config file, and %USERPROFILE% exists and is writeable. ([#73923](https://github.com/kubernetes/kubernetes/pull/73923), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery, SIG CLI, and SIG Windows",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73923",
-    "pr_number": 73923,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cli", "windows"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "74416": {
-    "commit": "7e17aeb1b5c817e37fbb36278004c63011be3544",
-    "text": "Add --shutdown-delay-duration to kube-apiserver in order to delay a graceful shutdown. `/healthz` will keep returning success during this time and requests are normally served, but `/readyz` will return faillure immediately. This delay can be used to allow the SDN to update iptables on all nodes and stop sending traffic.",
-    "markdown": "Add --shutdown-delay-duration to kube-apiserver in order to delay a graceful shutdown. `/healthz` will keep returning success during this time and requests are normally served, but `/readyz` will return faillure immediately. This delay can be used to allow the SDN to update iptables on all nodes and stop sending traffic. ([#74416](https://github.com/kubernetes/kubernetes/pull/74416), [@sttts](https://github.com/sttts))\n\n  Courtesy of SIG API Machinery",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74416",
-    "pr_number": 74416,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "74526": {
-    "commit": "5ece88c4c81ce40c0f248797f0951564d5bc7bf3",
-    "text": "When HPAScaleToZero feature gate is enabled HPA supports scaling to zero pods based on object or external metrics. HPA remains active as long as at least one metric value available.\n\nTo downgrade the cluster to version that does not support scale-to-zero feature:\n1. make sure there are no hpa objects with minReplicas=0. Here is a oneliner to update it to 1:\n    $ kubectl get hpa --all-namespaces  --no-headers=true | awk  '{if($6==0) printf \"kubectl patch hpa/%s --namespace=%s -p \\\"{\\\\\\\"spec\\\\\\\":{\\\\\\\"minReplicas\\\\\\\":1}}\\\"\\n\", $2, $1 }' | sh\n2. disable HPAScaleToZero feature gate",
-    "markdown": "When HPAScaleToZero feature gate is enabled HPA supports scaling to zero pods based on object or external metrics. HPA remains active as long as at least one metric value available.\n  \n  To downgrade the cluster to version that does not support scale-to-zero feature:\n  1. make sure there are no hpa objects with minReplicas=0. Here is a oneliner to update it to 1:\n      $ kubectl get hpa --all-namespaces  --no-headers=true | awk  '{if($6==0) printf \"kubectl patch hpa/%s --namespace=%s -p \\\"{\\\\\\\"spec\\\\\\\":{\\\\\\\"minReplicas\\\\\\\":1}}\\\"\\n\", $2, $1 }' | sh\n  2. disable HPAScaleToZero feature gate ([#74526](https://github.com/kubernetes/kubernetes/pull/74526), [@DXist](https://github.com/DXist))\n\n  Courtesy of SIG API Machinery, SIG Apps, and SIG Autoscaling",
-    "author": "DXist",
-    "author_url": "https://github.com/DXist",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74526",
-    "pr_number": 74526,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "autoscaling"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "74691": {
-    "commit": "514a157adff2dddf5928bdd8da3288e02ec9de49",
-    "text": "Use O_CLOEXEC to ensure file descriptors do not leak to subprocesses.",
-    "markdown": "Use O_CLOEXEC to ensure file descriptors do not leak to subprocesses. ([#74691](https://github.com/kubernetes/kubernetes/pull/74691), [@cpuguy83](https://github.com/cpuguy83))",
-    "author": "cpuguy83",
-    "author_url": "https://github.com/cpuguy83",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74691",
-    "pr_number": 74691,
-    "areas": ["dependency", "kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "75229": {
-    "commit": "e8559f7565863f3e852ec24cbd9f8322a80e5f30",
-    "text": "Attempt to set the kubelet's hostname \u0026 internal IP if `--cloud-provider=external` and no node addresses exists",
-    "markdown": "Attempt to set the kubelet's hostname \u0026 internal IP if `--cloud-provider=external` and no node addresses exists ([#75229](https://github.com/kubernetes/kubernetes/pull/75229), [@andrewsykim](https://github.com/andrewsykim))\n\n  Courtesy of SIG Node",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75229",
-    "pr_number": 75229,
-    "areas": ["cloudprovider", "kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "75593": {
-    "commit": "a61006bbc4af6f2d2fe1b254133ff7be4cb27476",
-    "text": "E2E tests no longer add command line flags directly to the command line, test suites that want that need to be updated if they don't use HandleFlags\nloading a -viper-config=e2e.yaml with suffix (introduced in 1.13) works again and now has a regression test",
-    "markdown": "E2E tests no longer add command line flags directly to the command line, test suites that want that need to be updated if they don't use HandleFlags\n  loading a -viper-config=e2e.yaml with suffix (introduced in 1.13) works again and now has a regression test ([#75593](https://github.com/kubernetes/kubernetes/pull/75593), [@pohly](https://github.com/pohly))",
-    "author": "pohly",
-    "author_url": "https://github.com/pohly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75593",
-    "pr_number": 75593,
-    "areas": ["e2e-test-framework", "kubeadm", "kubectl", "test"],
-    "kinds": ["bug"],
-    "sigs": ["cli", "cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "75805": {
-    "commit": "748849bbb92170c2acd6fe2a96c95a1362e7129a",
-    "text": "NFS Drivers are now enabled to collect metrics, StatFS metrics provider is used to collect the metrics. (\u0026#35;75805 , @brahmaroutu)",
-    "markdown": "NFS Drivers are now enabled to collect metrics, StatFS metrics provider is used to collect the metrics. (\u0026#35;75805 , @brahmaroutu) ([#75805](https://github.com/kubernetes/kubernetes/pull/75805), [@brahmaroutu](https://github.com/brahmaroutu))",
-    "author": "brahmaroutu",
-    "author_url": "https://github.com/brahmaroutu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75805",
-    "pr_number": 75805,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.5"
-  },
-  "76518": {
-    "commit": "920ac08361a0c96f2379bdeb3414003796be7856",
-    "text": "Limit the response body length of http readiness/liveness probes to 10KB;",
-    "markdown": "Limit the response body length of http readiness/liveness probes to 10KB; ([#76518](https://github.com/kubernetes/kubernetes/pull/76518), [@haiyanmeng](https://github.com/haiyanmeng))",
-    "author": "haiyanmeng",
-    "author_url": "https://github.com/haiyanmeng",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76518",
-    "pr_number": 76518,
-    "areas": ["apiserver", "cloudprovider", "dependency", "kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "network", "node"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "77327": {
-    "commit": "323356fda06cbac0b74c35f522f17c3bb9d580ca",
-    "text": "`[]TopologySpreadConstraint` is introduced into PodSpec to support the \"Even Pods Spread\" alpha feature.",
-    "markdown": "`[]TopologySpreadConstraint` is introduced into PodSpec to support the \"Even Pods Spread\" alpha feature. ([#77327](https://github.com/kubernetes/kubernetes/pull/77327), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77327",
-    "pr_number": 77327,
-    "areas": ["kubeadm", "kubelet", "test"],
-    "kinds": ["api-change"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "cluster-lifecycle",
-      "node",
-      "scheduling",
-      "storage",
-      "testing"
-    ],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "77355": {
-    "commit": "a172e197b165f60c215b4467175bdd61d8a15e0e",
-    "text": "Update gogo/protobuf to serialize backward, as to get better performance on deep objects.",
-    "markdown": "Update gogo/protobuf to serialize backward, as to get better performance on deep objects. ([#77355](https://github.com/kubernetes/kubernetes/pull/77355), [@apelisse](https://github.com/apelisse))",
-    "author": "apelisse",
-    "author_url": "https://github.com/apelisse",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77355",
-    "pr_number": 77355,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl", "kubelet"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery", "auth", "cli", "cloud-provider", "cluster-lifecycle", "node"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "77449": {
-    "commit": "7c7d70bc7c10f774e1bda021de8f337d075a5563",
-    "text": "Kubernetes now supports transparent compression of API responses. Clients that send `Accept-Encoding: gzip` will now receive a GZIP compressed response body if the API call was larger than 128KB.  Go clients automatically request gzip-encoding by default and should see reduced transfer times for very large API requests.  Clients in other languages may need to make changes to benefit from compression.",
-    "markdown": "Kubernetes now supports transparent compression of API responses. Clients that send `Accept-Encoding: gzip` will now receive a GZIP compressed response body if the API call was larger than 128KB.  Go clients automatically request gzip-encoding by default and should see reduced transfer times for very large API requests.  Clients in other languages may need to make changes to benefit from compression. ([#77449](https://github.com/kubernetes/kubernetes/pull/77449), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77449",
-    "pr_number": 77449,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery", "scalability"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "77561": {
-    "commit": "3be827e912d1db8570f653a05447e6e78a3a844b",
-    "text": "Use HTTPS as etcd-apiserver protocol when mTLS between etcd and kube-apiserver on master is enabled, change etcd metrics/health port to 2382.",
-    "markdown": "Use HTTPS as etcd-apiserver protocol when mTLS between etcd and kube-apiserver on master is enabled, change etcd metrics/health port to 2382. ([#77561](https://github.com/kubernetes/kubernetes/pull/77561), [@wenjiaswe](https://github.com/wenjiaswe))",
-    "author": "wenjiaswe",
-    "author_url": "https://github.com/wenjiaswe",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77561",
-    "pr_number": 77561,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cluster-lifecycle"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "78140": {
-    "commit": "416a717effeb3b0ff69e3363b15a9f6570a1a4a1",
-    "text": "Optimize EC2 DescribeInstances API calls in aws cloud provider library by querying instance ID instead of EC2 filters when possible",
-    "markdown": "Optimize EC2 DescribeInstances API calls in aws cloud provider library by querying instance ID instead of EC2 filters when possible ([#78140](https://github.com/kubernetes/kubernetes/pull/78140), [@zhan849](https://github.com/zhan849))\n\n  Courtesy of SIG Cloud Provider, and SIG Scalability",
-    "author": "zhan849",
-    "author_url": "https://github.com/zhan849",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78140",
-    "pr_number": 78140,
-    "areas": ["cloudprovider"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider", "scalability"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "78162": {
-    "commit": "c30f0248649c15a46ab99eb722de0448988198f8",
-    "text": "Add support for writing out of tree custom scheduler plugins.",
-    "markdown": "Add support for writing out of tree custom scheduler plugins. ([#78162](https://github.com/kubernetes/kubernetes/pull/78162), [@hex108](https://github.com/hex108))\n\n  Courtesy of SIG Scheduling",
-    "author": "hex108",
-    "author_url": "https://github.com/hex108",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78162",
-    "pr_number": 78162,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "78170": {
-    "commit": "bbdd0557b69e07e0cd1bf4377f725cafe9888b0d",
-    "text": "Fix a bug that causes DaemonSet rolling update hang when there exist failed pods.",
-    "markdown": "Fix a bug that causes DaemonSet rolling update hang when there exist failed pods. ([#78170](https://github.com/kubernetes/kubernetes/pull/78170), [@DaiHao](https://github.com/DaiHao))",
-    "author": "DaiHao",
-    "author_url": "https://github.com/DaiHao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78170",
-    "pr_number": 78170,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.14.5"
-  },
-  "78319": {
-    "commit": "ce80aeaac595e81361c051fe6a9a951ebe14d569",
-    "text": "Introduction of the pod overhead feature to the scheduler.  This functionality is alpha-level as of \nKubernetes v1.16, and is only honored by servers that enable the PodOverhead feature.gate.",
-    "markdown": "Introduction of the pod overhead feature to the scheduler.  This functionality is alpha-level as of \n  Kubernetes v1.16, and is only honored by servers that enable the PodOverhead feature.gate. ([#78319](https://github.com/kubernetes/kubernetes/pull/78319), [@egernst](https://github.com/egernst))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Node, and SIG Scheduling",
-    "author": "egernst",
-    "author_url": "https://github.com/egernst",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78319",
-    "pr_number": 78319,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "node", "scheduling"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "78447": {
-    "commit": "b0d08f6a47c7d3557e220634bcdb900398b4034b",
-    "text": "Migrate scheduler to use v1beta1 Event API. any tool targeting scheduler events needs to use v1beta1 Event API",
-    "markdown": "Migrate scheduler to use v1beta1 Event API. any tool targeting scheduler events needs to use v1beta1 Event API ([#78447](https://github.com/kubernetes/kubernetes/pull/78447), [@yastij](https://github.com/yastij))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG Scheduling, and SIG Testing",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78447",
-    "pr_number": 78447,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "apps", "auth", "scheduling", "testing"],
-    "feature": true,
-    "action_required": true,
-    "release_version": "1.14.5"
-  },
-  "78477": {
-    "commit": "f31d786927a7855b3dd2200d16e2b172661fb919",
-    "text": "Add Filter extension point to the scheduling framework.",
-    "markdown": "Add Filter extension point to the scheduling framework. ([#78477](https://github.com/kubernetes/kubernetes/pull/78477), [@YoubingLi](https://github.com/YoubingLi))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "YoubingLi",
-    "author_url": "https://github.com/YoubingLi",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78477",
-    "pr_number": 78477,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "78732": {
-    "commit": "74b5603f8326c7fcaca29e6a0d6827ca29863433",
-    "text": "Added a metric 'apiserver_watch_events_total' that can be used to understand the number of watch events in the system.",
-    "markdown": "Added a metric 'apiserver_watch_events_total' that can be used to understand the number of watch events in the system. ([#78732](https://github.com/kubernetes/kubernetes/pull/78732), [@mborsz](https://github.com/mborsz))\n\n  Courtesy of SIG API Machinery",
-    "author": "mborsz",
-    "author_url": "https://github.com/mborsz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78732",
-    "pr_number": 78732,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "78742": {
-    "commit": "91b26341f47088896ad373f4df8c7bdf5b249bcc",
-    "text": "The garbage collector and generic object quota controller have been updated to use the metadata client which improves memory\nand CPU usage of the Kube controller manager.",
-    "markdown": "The garbage collector and generic object quota controller have been updated to use the metadata client which improves memory\n  and CPU usage of the Kube controller manager. ([#78742](https://github.com/kubernetes/kubernetes/pull/78742), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78742",
-    "pr_number": 78742,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "apps", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "78744": {
-    "commit": "ca6cc69bd4e1e5b72b94863729a5a10936684867",
-    "text": "The namespace controller has been updated to use the metadata client which improves memory\nand CPU usage of the Kube controller manager.",
-    "markdown": "The namespace controller has been updated to use the metadata client which improves memory\n  and CPU usage of the Kube controller manager. ([#78744](https://github.com/kubernetes/kubernetes/pull/78744), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78744",
-    "pr_number": 78744,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "apps", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "78941": {
-    "commit": "fde94e931f90d3a0d01808caf0b2e94e9ea8c629",
-    "text": "Failed iscsi logout is now re-tried periodically.",
-    "markdown": "Failed iscsi logout is now re-tried periodically. ([#78941](https://github.com/kubernetes/kubernetes/pull/78941), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78941",
-    "pr_number": 78941,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "79014": {
-    "commit": "042b157d45ba671f77ffa14821b8a3b09b94127c",
-    "text": "updated fluentd to 1.5.1, elasticsearchs \u0026 kibana to 7.1.1",
-    "markdown": "updated fluentd to 1.5.1, elasticsearchs \u0026 kibana to 7.1.1 ([#79014](https://github.com/kubernetes/kubernetes/pull/79014), [@monotek](https://github.com/monotek))",
-    "author": "monotek",
-    "author_url": "https://github.com/monotek",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79014",
-    "pr_number": 79014,
-    "kinds": ["flake"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "79109": {
-    "commit": "a3898dc41db8f9d8d25ba95b3c92eb4e5d9e2d64",
-    "text": "Add Score extension point to the scheduling framework.",
-    "markdown": "Add Score extension point to the scheduling framework. ([#79109](https://github.com/kubernetes/kubernetes/pull/79109), [@ahg-g](https://github.com/ahg-g))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "ahg-g",
-    "author_url": "https://github.com/ahg-g",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79109",
-    "pr_number": 79109,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "79313": {
-    "commit": "fc5ef37db177158b7f0afce5bb59908803193bf7",
-    "text": "Add Bind extension point to the scheduling framework.",
-    "markdown": "Add Bind extension point to the scheduling framework. ([#79313](https://github.com/kubernetes/kubernetes/pull/79313), [@chenchun](https://github.com/chenchun))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "chenchun",
-    "author_url": "https://github.com/chenchun",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79313",
-    "pr_number": 79313,
-    "areas": ["test"],
-    "kinds": ["feature", "flake"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "79444": {
-    "commit": "1dac5fd14a54ac4972339dbe55f9f03688fd7542",
-    "text": "Reduce kube-proxy cpu usage in IPVS mode when a large number of nodePort services exist.",
-    "markdown": "Reduce kube-proxy cpu usage in IPVS mode when a large number of nodePort services exist. ([#79444](https://github.com/kubernetes/kubernetes/pull/79444), [@cezarsa](https://github.com/cezarsa))",
-    "author": "cezarsa",
-    "author_url": "https://github.com/cezarsa",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79444",
-    "pr_number": 79444,
-    "areas": ["ipvs"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.5"
-  },
-  "79489": {
-    "commit": "2b21e478b1414846f06fcc59cc99940c36da8187",
-    "text": "Add v1.Container.SecurityContext.WindowsOptions.RunAsUserName to the pod spec",
-    "markdown": "Add v1.Container.SecurityContext.WindowsOptions.RunAsUserName to the pod spec ([#79489](https://github.com/kubernetes/kubernetes/pull/79489), [@claudiubelu](https://github.com/claudiubelu))",
-    "author": "claudiubelu",
-    "author_url": "https://github.com/claudiubelu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79489",
-    "pr_number": 79489,
-    "areas": ["kubelet", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["apps", "node", "testing", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "79549": {
-    "commit": "3dd8add5d25d29358fb239ac23461643f1d8effc",
-    "text": "The `MutatingWebhookConfiguration` and `ValidatingWebhookConfiguration` APIs have been promoted to `admissionregistration.k8s.io/v1`:\n- `failurePolicy` default changed from `Ignore` to `Fail` for v1\n- `matchPolicy` default changed from `Exact` to `Equivalent` for v1\n- `timeout` default changed from `30s` to `10s` for v1\n- `sideEffects` default value is removed, and the field made required, and only `None` and `NoneOnDryRun` are permitted for v1\n- `admissionReviewVersions` default value is removed and the field made required for v1 (supported versions for AdmissionReview are `v1` and `v1beta1`)\n- The `name` field for specified webhooks must be unique for `MutatingWebhookConfiguration` and `ValidatingWebhookConfiguration` objects created via `admissionregistration.k8s.io/v1`\nThe `admissionregistration.k8s.io/v1beta1` versions of `MutatingWebhookConfiguration` and `ValidatingWebhookConfiguration` are deprecated and will no longer be served in v1.19.",
-    "markdown": "The `MutatingWebhookConfiguration` and `ValidatingWebhookConfiguration` APIs have been promoted to `admissionregistration.k8s.io/v1`:\n  - `failurePolicy` default changed from `Ignore` to `Fail` for v1\n  - `matchPolicy` default changed from `Exact` to `Equivalent` for v1\n  - `timeout` default changed from `30s` to `10s` for v1\n  - `sideEffects` default value is removed, and the field made required, and only `None` and `NoneOnDryRun` are permitted for v1\n  - `admissionReviewVersions` default value is removed and the field made required for v1 (supported versions for AdmissionReview are `v1` and `v1beta1`)\n  - The `name` field for specified webhooks must be unique for `MutatingWebhookConfiguration` and `ValidatingWebhookConfiguration` objects created via `admissionregistration.k8s.io/v1`\n  The `admissionregistration.k8s.io/v1beta1` versions of `MutatingWebhookConfiguration` and `ValidatingWebhookConfiguration` are deprecated and will no longer be served in v1.19. ([#79549](https://github.com/kubernetes/kubernetes/pull/79549), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG CLI, and SIG Testing",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79549",
-    "pr_number": 79549,
-    "areas": ["admission-control", "dependency", "kubectl", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "cli", "testing"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "79552": {
-    "commit": "60c2d44129ba238ceb548467b3931c9b76032965",
-    "text": "Add support for AWS EBS on windows",
-    "markdown": "Add support for AWS EBS on windows ([#79552](https://github.com/kubernetes/kubernetes/pull/79552), [@wongma7](https://github.com/wongma7))\n\n  Courtesy of SIG Storage, SIG Testing, and SIG Windows",
-    "author": "wongma7",
-    "author_url": "https://github.com/wongma7",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79552",
-    "pr_number": 79552,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["storage", "testing", "windows"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "79554": {
-    "commit": "642a06e552d1b38fa2590b89d8352b1c2871b1a1",
-    "text": "scheduler.alpha.kubernetes.io/critical-pod annotation is removed. Pod priority (spec.priorityClassName) should be used instead to mark pods as critical. Action required!",
-    "markdown": "scheduler.alpha.kubernetes.io/critical-pod annotation is removed. Pod priority (spec.priorityClassName) should be used instead to mark pods as critical. Action required! ([#79554](https://github.com/kubernetes/kubernetes/pull/79554), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Apps, SIG Cluster Lifecycle, SIG Instrumentation, SIG Node, SIG Scheduling, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79554",
-    "pr_number": 79554,
-    "areas": ["kubelet", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["apps", "cluster-lifecycle", "instrumentation", "node", "scheduling", "testing"],
-    "action_required": true,
-    "release_version": "1.14.5"
-  },
-  "79677": {
-    "commit": "7340b6341a4e867cfc5761e8642c54c55ff6db3e",
-    "text": "kubeadm: implement support for concurrent add/remove of stacked etcd members",
-    "markdown": "kubeadm: implement support for concurrent add/remove of stacked etcd members ([#79677](https://github.com/kubernetes/kubernetes/pull/79677), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79677",
-    "pr_number": 79677,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.5"
-  },
-  "79724": {
-    "commit": "b130ff450232eaf9100548aa08295168871a4ee9",
-    "text": "add  `kubectl replace --raw` and `kubectl delete --raw` to have parity with create and get",
-    "markdown": "add  `kubectl replace --raw` and `kubectl delete --raw` to have parity with create and get ([#79724](https://github.com/kubernetes/kubernetes/pull/79724), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79724",
-    "pr_number": 79724,
-    "areas": ["dependency", "kubectl", "release-eng", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cli", "release", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "79786": {
-    "commit": "a46eed0ca3988a8d1fba78e2b8f882df1f7406f4",
-    "text": "Promote WatchBookmark feature to beta and enable it by default.\nWith WatchBookmark feature, clients are able to request watch events with BOOKMARK type. Clients should not assume bookmarks are returned at any specific interval, nor may they assume the server will send any BOOKMARK event during a session.",
-    "markdown": "Promote WatchBookmark feature to beta and enable it by default.\n  With WatchBookmark feature, clients are able to request watch events with BOOKMARK type. Clients should not assume bookmarks are returned at any specific interval, nor may they assume the server will send any BOOKMARK event during a session. ([#79786](https://github.com/kubernetes/kubernetes/pull/79786), [@wojtek-t](https://github.com/wojtek-t))\n\n  Courtesy of SIG API Machinery",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79786",
-    "pr_number": 79786,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "79843": {
-    "commit": "6cd0392ced5a14409c0dd074bdb8b9c33a433f06",
-    "text": "The sample-apiserver gains support for OpenAPI v2 spec serving at `/openapi/v2`.\n\nThe `generate-internal-groups.sh` script in k8s.io/code-generator will generate OpenAPI definitions by default in `pkg/generated/openapi`. Additional API group dependencies can be added via `OPENAPI_EXTRA_PACKAGES=\u003cgroup\u003e/\u003cversion\u003e \u003cgroup2\u003e/\u003cversion2\u003e...`.",
-    "markdown": "The sample-apiserver gains support for OpenAPI v2 spec serving at `/openapi/v2`.\n  \n  The `generate-internal-groups.sh` script in k8s.io/code-generator will generate OpenAPI definitions by default in `pkg/generated/openapi`. Additional API group dependencies can be added via `OPENAPI_EXTRA_PACKAGES=\u003cgroup\u003e/\u003cversion\u003e \u003cgroup2\u003e/\u003cversion2\u003e...`. ([#79843](https://github.com/kubernetes/kubernetes/pull/79843), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79843",
-    "pr_number": 79843,
-    "areas": ["code-generation", "dependency", "release-eng", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "release", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "79851": {
-    "commit": "0e3b593ded745f0aac2261cd9f8938e0d6f5556c",
-    "text": "Fixed a bug in the CSI metrics that does not return not supported error when a CSI driver does not support metrics.",
-    "markdown": "Fixed a bug in the CSI metrics that does not return not supported error when a CSI driver does not support metrics. ([#79851](https://github.com/kubernetes/kubernetes/pull/79851), [@jparklab](https://github.com/jparklab))",
-    "author": "jparklab",
-    "author_url": "https://github.com/jparklab",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79851",
-    "pr_number": 79851,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.5"
-  },
-  "79895": {
-    "commit": "262e59b2c04bf71ec4e56fdc2f3a91a5878336f4",
-    "text": "Resolves an issue serving aggregated APIs backed by services that respond to requests to `/` with non-2xx HTTP responses",
-    "markdown": "Resolves an issue serving aggregated APIs backed by services that respond to requests to `/` with non-2xx HTTP responses ([#79895](https://github.com/kubernetes/kubernetes/pull/79895), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79895",
-    "pr_number": 79895,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.5"
-  },
-  "79920": {
-    "commit": "10a1d1f5bdeb1a41438814489e2e594ab2805363",
-    "text": "Bugfix: csi plugin supporting raw block that does not need attach mounted failed",
-    "markdown": "Bugfix: csi plugin supporting raw block that does not need attach mounted failed ([#79920](https://github.com/kubernetes/kubernetes/pull/79920), [@cwdsuzhou](https://github.com/cwdsuzhou))",
-    "author": "cwdsuzhou",
-    "author_url": "https://github.com/cwdsuzhou",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79920",
-    "pr_number": 79920,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.5"
-  },
-  "79966": {
-    "commit": "824dfe93899e1d6da4ee6965db52f83eb56e06ac",
-    "text": "update to use go 1.12.7",
-    "markdown": "update to use go 1.12.7 ([#79966](https://github.com/kubernetes/kubernetes/pull/79966), [@tao12345666333](https://github.com/tao12345666333))\n\n  Courtesy of SIG Testing",
-    "author": "tao12345666333",
-    "author_url": "https://github.com/tao12345666333",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79966",
-    "pr_number": 79966,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["testing"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "79983": {
-    "commit": "a3750501b081ecd3ca33df496575066da7449140",
-    "text": "when PodInfoOnMount is enabled for a CSI driver, the new csi.storage.k8s.io/ephemeral parameter in the volume context allows a driver's NodePublishVolume implementation to determine on a case-by-case basis whether the volume is ephemeral or a normal persistent volume",
-    "markdown": "when PodInfoOnMount is enabled for a CSI driver, the new csi.storage.k8s.io/ephemeral parameter in the volume context allows a driver's NodePublishVolume implementation to determine on a case-by-case basis whether the volume is ephemeral or a normal persistent volume ([#79983](https://github.com/kubernetes/kubernetes/pull/79983), [@pohly](https://github.com/pohly))\n\n  Courtesy of SIG Storage, and SIG Testing",
-    "author": "pohly",
-    "author_url": "https://github.com/pohly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79983",
-    "pr_number": 79983,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["storage", "testing"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "79992": {
-    "commit": "e0ff9634d5b5a480595930236c87dec1327dbf13",
-    "text": "CRDs under k8s.io and kubernetes.io must have the \"api-approved.kubernetes.io\" set to either `unapproved.*` or a link to the pull request approving the schema.  See https://github.com/kubernetes/enhancements/pull/1111 for more details.",
-    "markdown": "CRDs under k8s.io and kubernetes.io must have the \"api-approved.kubernetes.io\" set to either `unapproved.*` or a link to the pull request approving the schema.  See https://github.com/kubernetes/enhancements/pull/1111 for more details. ([#79992](https://github.com/kubernetes/kubernetes/pull/79992), [@deads2k](https://github.com/deads2k))\n\n  Courtesy of SIG API Machinery",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79992",
-    "pr_number": 79992,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "80007": {
-    "commit": "ab960c612c4ca804fac2548a0f72bfd00761860c",
-    "text": "SubjectAccessReview requests sent for RBAC escalation, impersonation, and pod security policy authorization checks now populate the version attribute.",
-    "markdown": "SubjectAccessReview requests sent for RBAC escalation, impersonation, and pod security policy authorization checks now populate the version attribute. ([#80007](https://github.com/kubernetes/kubernetes/pull/80007), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80007",
-    "pr_number": 80007,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "auth"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "80017": {
-    "commit": "7e4a6ca3f77fe8cb33a788fee98a148327310dfd",
-    "text": "hyperkube: the `--make-symlinks` flag, deprecated in v1.14, has been removed.",
-    "markdown": "hyperkube: the `--make-symlinks` flag, deprecated in v1.14, has been removed. ([#80017](https://github.com/kubernetes/kubernetes/pull/80017), [@Pothulapati](https://github.com/Pothulapati))\n\n  Courtesy of SIG Release",
-    "author": "Pothulapati",
-    "author_url": "https://github.com/Pothulapati",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80017",
-    "pr_number": 80017,
-    "kinds": ["cleanup"],
-    "sigs": ["release"],
-    "action_required": true,
-    "release_version": "1.14.5"
-  },
-  "80024": {
-    "commit": "65fc256cc3ddd279849444c03c792d98d58f34ff",
-    "text": "kubeadm: fall back to client version in case of certain HTTP errors",
-    "markdown": "kubeadm: fall back to client version in case of certain HTTP errors ([#80024](https://github.com/kubernetes/kubernetes/pull/80024), [@RainbowMango](https://github.com/RainbowMango))",
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80024",
-    "pr_number": 80024,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.5"
-  },
-  "80025": {
-    "commit": "ae676161105fb88616850f9596021719df6d0106",
-    "text": "kubeadm: support fetching configuration from the original cluster for 'upgrade diff'",
-    "markdown": "kubeadm: support fetching configuration from the original cluster for 'upgrade diff' ([#80025](https://github.com/kubernetes/kubernetes/pull/80025), [@SataQiu](https://github.com/SataQiu))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80025",
-    "pr_number": 80025,
-    "areas": ["kubeadm"],
-    "kinds": ["bug", "cleanup", "feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "80037": {
-    "commit": "aaad86c2105bfbf98f26745bf8262ab8c357e8d3",
-    "text": "etcd migration image no longer supports etcd2 version.",
-    "markdown": "etcd migration image no longer supports etcd2 version. ([#80037](https://github.com/kubernetes/kubernetes/pull/80037), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80037",
-    "pr_number": 80037,
-    "areas": ["code-organization", "dependency"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cluster-lifecycle"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "80043": {
-    "commit": "676417ec7bacca7b277aa2504667cd2c11633af0",
-    "text": "kubelet's --containerized flag was deprecated in 1.14. This flag is removed in 1.16.",
-    "markdown": "kubelet's --containerized flag was deprecated in 1.14. This flag is removed in 1.16. ([#80043](https://github.com/kubernetes/kubernetes/pull/80043), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80043",
-    "pr_number": 80043,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.14.5"
-  },
-  "80099": {
-    "commit": "844a0c9a10cd52367ec7c05f5a8c176535dcef38",
-    "text": "Cinder and ScaleIO volume providers have been deprecated and will be removed in a future release.",
-    "markdown": "Cinder and ScaleIO volume providers have been deprecated and will be removed in a future release. ([#80099](https://github.com/kubernetes/kubernetes/pull/80099), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80099",
-    "pr_number": 80099,
-    "kinds": ["cleanup"],
-    "sigs": ["storage"],
-    "release_version": "1.14.5"
-  },
-  "80100": {
-    "commit": "87b744715ec6952c45d04253dc7b63fc3cfe1ddc",
-    "text": "Increase log level for graceful termination to v=5",
-    "markdown": "Increase log level for graceful termination to v=5 ([#80100](https://github.com/kubernetes/kubernetes/pull/80100), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80100",
-    "pr_number": 80100,
-    "areas": ["ipvs"],
-    "kinds": ["cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.14.5"
-  },
-  "80115": {
-    "commit": "ab8506fb9b57671da54c9104cc080370592c0f80",
-    "text": "Add --kubernetes-version to \"kubeadm init phase certs ca\" and \"kubeadm init phase kubeconfig\"",
-    "markdown": "Add --kubernetes-version to \"kubeadm init phase certs ca\" and \"kubeadm init phase kubeconfig\" ([#80115](https://github.com/kubernetes/kubernetes/pull/80115), [@gyuho](https://github.com/gyuho))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "gyuho",
-    "author_url": "https://github.com/gyuho",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80115",
-    "pr_number": 80115,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "80142": {
-    "commit": "fe0404171c9e33daf1aa0798dbeae0f40df1e3a0",
-    "text": "Remove deprecated github.com/kardianos/osext dependency",
-    "markdown": "Remove deprecated github.com/kardianos/osext dependency ([#80142](https://github.com/kubernetes/kubernetes/pull/80142), [@loqutus](https://github.com/loqutus))",
-    "author": "loqutus",
-    "author_url": "https://github.com/loqutus",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80142",
-    "pr_number": 80142,
-    "areas": ["dependency", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["testing"],
-    "release_version": "1.14.5"
-  },
-  "80145": {
-    "commit": "768acf600943442d517d16fdaaa786032099c995",
-    "text": "kubeadm ClusterConfiguration now supports featureGates: IPv6DualStack: true",
-    "markdown": "kubeadm ClusterConfiguration now supports featureGates: IPv6DualStack: true ([#80145](https://github.com/kubernetes/kubernetes/pull/80145), [@Arvinderpal](https://github.com/Arvinderpal))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "Arvinderpal",
-    "author_url": "https://github.com/Arvinderpal",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80145",
-    "pr_number": 80145,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "80157": {
-    "commit": "635b83c5f4065c5528dea2697c605cda969dd2cc",
-    "text": "Fix a bug that ListOptions.AllowWatchBookmarks wasn't propagating correctly in kube-apiserver.",
-    "markdown": "Fix a bug that ListOptions.AllowWatchBookmarks wasn't propagating correctly in kube-apiserver. ([#80157](https://github.com/kubernetes/kubernetes/pull/80157), [@wojtek-t](https://github.com/wojtek-t))",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80157",
-    "pr_number": 80157,
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery", "scalability"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "80173": {
-    "commit": "ab266f725ff576b067750b2ae36f41b08f8cb45a",
-    "text": "make node lease renew interval more heuristic based on node-status-update-frequency in kubelet",
-    "markdown": "make node lease renew interval more heuristic based on node-status-update-frequency in kubelet ([#80173](https://github.com/kubernetes/kubernetes/pull/80173), [@gaorong](https://github.com/gaorong))",
-    "author": "gaorong",
-    "author_url": "https://github.com/gaorong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80173",
-    "pr_number": 80173,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.5"
-  },
-  "80176": {
-    "commit": "04c9bd9bfa420d369e131f7bc80eb3e212284bbe",
-    "text": "cpuUsageNanoCores is now reported in the Kubelet summary API on Windows nodes",
-    "markdown": "cpuUsageNanoCores is now reported in the Kubelet summary API on Windows nodes ([#80176](https://github.com/kubernetes/kubernetes/pull/80176), [@liyanhui1228](https://github.com/liyanhui1228))",
-    "author": "liyanhui1228",
-    "author_url": "https://github.com/liyanhui1228",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80176",
-    "pr_number": 80176,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "80191": {
-    "commit": "835552ecb6626e9751e52d22c6afbc3e4aa1231f",
-    "text": "Pass-through volume MountOptions to global mount (NodeStageVolume) on the node for CSI",
-    "markdown": "Pass-through volume MountOptions to global mount (NodeStageVolume) on the node for CSI ([#80191](https://github.com/kubernetes/kubernetes/pull/80191), [@davidz627](https://github.com/davidz627))",
-    "author": "davidz627",
-    "author_url": "https://github.com/davidz627",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80191",
-    "pr_number": 80191,
-    "areas": ["test"],
-    "kinds": ["bug", "failing-test"],
-    "sigs": ["storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "80266": {
-    "commit": "5de41340ce3917ab948573f6358b10512a0333ce",
-    "text": "container images tar files for 'amd64' will now contain the architecture in the RepoTags manifest.json section.\nIf you are using docker manifests there are not visible changes.",
-    "markdown": "container images tar files for 'amd64' will now contain the architecture in the RepoTags manifest.json section.\n  If you are using docker manifests there are not visible changes. ([#80266](https://github.com/kubernetes/kubernetes/pull/80266), [@javier-b-perez](https://github.com/javier-b-perez))\n\n  Courtesy of SIG Release",
-    "author": "javier-b-perez",
-    "author_url": "https://github.com/javier-b-perez",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80266",
-    "pr_number": 80266,
-    "kinds": ["bug"],
-    "sigs": ["release"],
-    "action_required": true,
-    "release_version": "1.14.5"
-  },
-  "80277": {
-    "commit": "49f6510d9af15e4cf0b70fc2011f5db956258bbe",
-    "text": "Revert \"scheduler.alpha.kubernetes.io/critical-pod annotation is removed. Pod priority (spec.priorityClassName) should be used instead to mark pods as critical. Action required!\"",
-    "markdown": "Revert \"scheduler.alpha.kubernetes.io/critical-pod annotation is removed. Pod priority (spec.priorityClassName) should be used instead to mark pods as critical. Action required!\" ([#80277](https://github.com/kubernetes/kubernetes/pull/80277), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Apps, SIG Cluster Lifecycle, SIG Instrumentation, SIG Node, SIG Scheduling, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80277",
-    "pr_number": 80277,
-    "areas": ["kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "cluster-lifecycle", "instrumentation", "node", "scheduling", "testing"],
-    "action_required": true,
-    "release_version": "1.14.5"
-  },
-  "80294": {
-    "commit": "304e2c247ae7566279fdba73895a24221ed583d8",
-    "text": "Passing an invalid policy name in the `--cpu-manager-policy` flag will now cause the kubelet to fail instead of simply ignoring the flag and running the `cpumanager`s default policy instead.",
-    "markdown": "Passing an invalid policy name in the `--cpu-manager-policy` flag will now cause the kubelet to fail instead of simply ignoring the flag and running the `cpumanager`s default policy instead. ([#80294](https://github.com/kubernetes/kubernetes/pull/80294), [@klueska](https://github.com/klueska))",
-    "author": "klueska",
-    "author_url": "https://github.com/klueska",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80294",
-    "pr_number": 80294,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.14.5"
-  },
-  "80361": {
-    "commit": "69a34f6a6f67de47cb9b72b6ac98e089d301beb3",
-    "text": "Remove GetReference() and GetPartialReference() function from pkg/api/ref, as the same function exists also in staging/src/k8s.io/client-go/tools/ref",
-    "markdown": "Remove GetReference() and GetPartialReference() function from pkg/api/ref, as the same function exists also in staging/src/k8s.io/client-go/tools/ref ([#80361](https://github.com/kubernetes/kubernetes/pull/80361), [@wojtek-t](https://github.com/wojtek-t))",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80361",
-    "pr_number": 80361,
-    "areas": ["code-organization"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.5"
-  },
-  "80383": {
-    "commit": "fb46ec845528dae0796a581583fd4fcdc764c104",
-    "text": "Implement normalize plugin extension point for the scheduler framework.",
-    "markdown": "Implement normalize plugin extension point for the scheduler framework. ([#80383](https://github.com/kubernetes/kubernetes/pull/80383), [@liu-cong](https://github.com/liu-cong))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Architecture, SIG Auth, SIG Autoscaling, SIG CLI, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Instrumentation, SIG Network, SIG Node, SIG Release, SIG Scheduling, SIG Storage, and SIG Testing",
-    "author": "liu-cong",
-    "author_url": "https://github.com/liu-cong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80383",
-    "pr_number": 80383,
-    "areas": [
-      "apiserver",
-      "cloudprovider",
-      "code-generation",
-      "conformance",
-      "dependency",
-      "e2e-test-framework",
-      "kubeadm",
-      "kubectl",
-      "kubelet",
-      "release-eng",
-      "test"
-    ],
-    "kinds": ["api-change", "feature"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "architecture",
-      "auth",
-      "autoscaling",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "network",
-      "node",
-      "release",
-      "scheduling",
-      "storage",
-      "testing"
-    ],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "80419": {
-    "commit": "c08a88a2d14c26a5376149c6a3301db410cd5ca4",
-    "text": "Fix retry issues when the nodes are under deleting on Azure",
-    "markdown": "Fix retry issues when the nodes are under deleting on Azure ([#80419](https://github.com/kubernetes/kubernetes/pull/80419), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80419",
-    "pr_number": 80419,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.5"
-  },
-  "80421": {
-    "commit": "10005d2e1e1425904f8c7bf5615e730fb0fea7c9",
-    "text": "Kibana has been slightly revamped/improved in the latest version",
-    "markdown": "Kibana has been slightly revamped/improved in the latest version ([#80421](https://github.com/kubernetes/kubernetes/pull/80421), [@lostick](https://github.com/lostick))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Instrumentation",
-    "author": "lostick",
-    "author_url": "https://github.com/lostick",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80421",
-    "pr_number": 80421,
-    "areas": ["dependency"],
-    "kinds": ["bug", "feature"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "80429": {
-    "commit": "f18833ee582d9f3bbe57e5f7c97f933c455640dc",
-    "text": "kubelet: change node-lease-renew-interval to 0.25 of lease-renew-duration",
-    "markdown": "kubelet: change node-lease-renew-interval to 0.25 of lease-renew-duration ([#80429](https://github.com/kubernetes/kubernetes/pull/80429), [@gaorong](https://github.com/gaorong))\n\n  Courtesy of SIG Architecture, and SIG Node",
-    "author": "gaorong",
-    "author_url": "https://github.com/gaorong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80429",
-    "pr_number": 80429,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["architecture", "node"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "80436": {
-    "commit": "08f9f2b58a6aa4924e9083f4e04735858e1c68db",
-    "text": "Fix CVE-2019-11249: Incomplete fixes for CVE-2019-1002101 and CVE-2019-11246, kubectl cp potential directory traversal",
-    "markdown": "Fix CVE-2019-11249: Incomplete fixes for CVE-2019-1002101 and CVE-2019-11246, kubectl cp potential directory traversal ([#80436](https://github.com/kubernetes/kubernetes/pull/80436), [@M00nF1sh](https://github.com/M00nF1sh))",
-    "author": "M00nF1sh",
-    "author_url": "https://github.com/M00nF1sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80436",
-    "pr_number": 80436,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.5"
-  },
-  "80443": {
-    "commit": "030274eba30668968ecfb43cf5ef3dfc642c3f4d",
-    "text": "Add CSI Migration Shim for VerifyVolumesAreAttached and BulkVolumeVerify",
-    "markdown": "Add CSI Migration Shim for VerifyVolumesAreAttached and BulkVolumeVerify ([#80443](https://github.com/kubernetes/kubernetes/pull/80443), [@davidz627](https://github.com/davidz627))\n\n  Courtesy of SIG Storage",
-    "author": "davidz627",
-    "author_url": "https://github.com/davidz627",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80443",
-    "pr_number": 80443,
-    "kinds": ["feature"],
-    "sigs": ["storage"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "80495": {
-    "commit": "de73f62336c9dcc80ef87a46f8c0127ddf6d1c6c",
-    "text": "Update portworx plugin dependency on libopenstorage/openstorage to v1.0.0",
-    "markdown": "Update portworx plugin dependency on libopenstorage/openstorage to v1.0.0 ([#80495](https://github.com/kubernetes/kubernetes/pull/80495), [@adityadani](https://github.com/adityadani))",
-    "author": "adityadani",
-    "author_url": "https://github.com/adityadani",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80495",
-    "pr_number": 80495,
-    "areas": ["code-organization", "dependency"],
-    "kinds": ["cleanup"],
-    "sigs": ["storage"],
-    "release_version": "1.14.5"
-  },
-  "80509": {
-    "commit": "44de947f0f86c0b2558b4ef0872801bd258a5949",
-    "text": "New flag --endpoint-updates-batch-period in kube-controller-manager can be used to reduce number of endpoints updates generated by pod changes.",
-    "markdown": "New flag --endpoint-updates-batch-period in kube-controller-manager can be used to reduce number of endpoints updates generated by pod changes. ([#80509](https://github.com/kubernetes/kubernetes/pull/80509), [@mborsz](https://github.com/mborsz))\n\n  Courtesy of SIG Apps, and SIG Network",
-    "author": "mborsz",
-    "author_url": "https://github.com/mborsz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80509",
-    "pr_number": 80509,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["apps", "network"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "80518": {
-    "commit": "1a95199311a2ad72cf6f85de19f25e616869e207",
-    "text": "Fixed detachment of deleted volumes on OpenStack / Cinder.",
-    "markdown": "Fixed detachment of deleted volumes on OpenStack / Cinder. ([#80518](https://github.com/kubernetes/kubernetes/pull/80518), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80518",
-    "pr_number": 80518,
-    "areas": ["cloudprovider", "provider/openstack"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "80529": {
-    "commit": "feb40b844c12e0b7e528bbcdaa41f218b3f5b881",
-    "text": "kubeadm: fixed ignoring errors when pulling control plane images",
-    "markdown": "kubeadm: fixed ignoring errors when pulling control plane images ([#80529](https://github.com/kubernetes/kubernetes/pull/80529), [@bart0sh](https://github.com/bart0sh))",
-    "author": "bart0sh",
-    "author_url": "https://github.com/bart0sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80529",
-    "pr_number": 80529,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.5"
-  },
-  "80531": {
-    "commit": "5bfa3664fb8ad22a784a8e0ce4107dd4146b029e",
-    "text": "In order to enable dual-stack support within kubeadm and kubernetes components, as part of the init config file, the user should set feature-gate IPv6DualStack=true in the ClusterConfiguration. Additionally, for each worker node, the user should set the feature-gate for kubelet using either nodeRegistration.kubeletExtraArgs or  KUBELET_EXTRA_ARGS.",
-    "markdown": "In order to enable dual-stack support within kubeadm and kubernetes components, as part of the init config file, the user should set feature-gate IPv6DualStack=true in the ClusterConfiguration. Additionally, for each worker node, the user should set the feature-gate for kubelet using either nodeRegistration.kubeletExtraArgs or  KUBELET_EXTRA_ARGS. ([#80531](https://github.com/kubernetes/kubernetes/pull/80531), [@Arvinderpal](https://github.com/Arvinderpal))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "Arvinderpal",
-    "author_url": "https://github.com/Arvinderpal",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80531",
-    "pr_number": 80531,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.5"
-  },
-  "80573": {
-    "commit": "8f1eeacb78aa99c5e8d93998912cc94cf8215c44",
-    "text": "kubeadm: produce errors if they occur when resetting cluster status for a control-plane node",
-    "markdown": "kubeadm: produce errors if they occur when resetting cluster status for a control-plane node ([#80573](https://github.com/kubernetes/kubernetes/pull/80573), [@bart0sh](https://github.com/bart0sh))",
-    "author": "bart0sh",
-    "author_url": "https://github.com/bart0sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80573",
-    "pr_number": 80573,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.5"
-  },
-  "80575": {
-    "commit": "a1f4c2ed21ea008c59934a9c6a06fac3da72f963",
-    "text": "Fixed a bug in kube-addon-manager's leader election logic that made all replicas active.",
-    "markdown": "Fixed a bug in kube-addon-manager's leader election logic that made all replicas active. ([#80575](https://github.com/kubernetes/kubernetes/pull/80575), [@mborsz](https://github.com/mborsz))",
-    "author": "mborsz",
-    "author_url": "https://github.com/mborsz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80575",
-    "pr_number": 80575,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.5"
-  },
-  "80591": {
-    "commit": "0775e6b2db031f84b8e7116547b8b5cbe0cfd51a",
-    "text": "Fixes problems with connecting to services on localhost on some systems; in particular, DNS queries to systemd-resolved on Ubuntu.",
-    "markdown": "Fixes problems with connecting to services on localhost on some systems; in particular, DNS queries to systemd-resolved on Ubuntu. ([#80591](https://github.com/kubernetes/kubernetes/pull/80591), [@danwinship](https://github.com/danwinship))",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80591",
-    "pr_number": 80591,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["network", "node"],
-    "duplicate": true,
-    "release_version": "1.14.5"
-  },
-  "80624": {
-    "commit": "78915a10949e74feb77586f71ee035292462ea56",
-    "text": "When a load balancer type service is created in a k8s cluster that is backed by Azure Standard Load Balancer, the corresponding load balancer rule added in the Azure Standard Load Balancer would now have the \"EnableTcpReset\" property set to true.",
-    "markdown": "When a load balancer type service is created in a k8s cluster that is backed by Azure Standard Load Balancer, the corresponding load balancer rule added in the Azure Standard Load Balancer would now have the \"EnableTcpReset\" property set to true. ([#80624](https://github.com/kubernetes/kubernetes/pull/80624), [@xuto2](https://github.com/xuto2))",
-    "author": "xuto2",
-    "author_url": "https://github.com/xuto2",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80624",
-    "pr_number": 80624,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.5"
-  },
-  "80648": {
-    "commit": "9bae1bc56804db4905abebcd408e0f02e199ab93",
-    "text": "Fix error handling and potential go null pointer exception in kubeadm upgrade diff",
-    "markdown": "Fix error handling and potential go null pointer exception in kubeadm upgrade diff ([#80648](https://github.com/kubernetes/kubernetes/pull/80648), [@odinuge](https://github.com/odinuge))",
-    "author": "odinuge",
-    "author_url": "https://github.com/odinuge",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80648",
-    "pr_number": 80648,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.5"
-  },
-  "80675": {
-    "commit": "000932d5844c2d2e01f99872e3df268cf7c25d14",
-    "text": "Fix error in `kubeadm join --discovery-file` when using discovery files with embedded credentials",
-    "markdown": "Fix error in `kubeadm join --discovery-file` when using discovery files with embedded credentials ([#80675](https://github.com/kubernetes/kubernetes/pull/80675), [@fabriziopandini](https://github.com/fabriziopandini))",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80675",
-    "pr_number": 80675,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.5"
-  },
-  "80676": {
-    "commit": "82a252ad746d0230519ee2da78b6b5fef4896fde",
-    "text": "kubeadm now deletes the bootstrap-kubelet.conf file after TLS bootstrap\nUser relying on bootstrap-kubelet.conf should switch to kubelet.conf that contains node credentials",
-    "markdown": "kubeadm now deletes the bootstrap-kubelet.conf file after TLS bootstrap\n  User relying on bootstrap-kubelet.conf should switch to kubelet.conf that contains node credentials ([#80676](https://github.com/kubernetes/kubernetes/pull/80676), [@fabriziopandini](https://github.com/fabriziopandini))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80676",
-    "pr_number": 80676,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "action_required": true,
-    "release_version": "1.14.5"
-  },
-  "80703": {
-    "commit": "5413252e47fcb9b05059bc2061786e25575b6cd3",
-    "text": "Fix public IP not found issues for VMSS nodes",
-    "markdown": "Fix public IP not found issues for VMSS nodes ([#80703](https://github.com/kubernetes/kubernetes/pull/80703), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80703",
-    "pr_number": 80703,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.5"
-  },
-  "80750": {
-    "commit": "2abc859540cca6124828614facb491356964fc1f",
-    "text": "Fix CVE-2019-11247: API server allows access to custom resources via wrong scope",
-    "markdown": "Fix CVE-2019-11247: API server allows access to custom resources via wrong scope ([#80750](https://github.com/kubernetes/kubernetes/pull/80750), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80750",
-    "pr_number": 80750,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.5"
-  },
-  "80798": {
-    "commit": "3758426884e3c82cbd99c72e8015f4396f21fde2",
-    "text": "kubeadm: avoid double deletion of the upgrade prepull DaemonSet",
-    "markdown": "kubeadm: avoid double deletion of the upgrade prepull DaemonSet ([#80798](https://github.com/kubernetes/kubernetes/pull/80798), [@xlgao-zju](https://github.com/xlgao-zju))",
-    "author": "xlgao-zju",
-    "author_url": "https://github.com/xlgao-zju",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80798",
-    "pr_number": 80798,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.5"
-  },
-  "80802": {
-    "commit": "c91761da0d854aa842ea55ba4d7c6cbc8d675892",
-    "text": "Fixed the bash completion error with override flags.",
-    "markdown": "Fixed the bash completion error with override flags. ([#80802](https://github.com/kubernetes/kubernetes/pull/80802), [@dtaniwaki](https://github.com/dtaniwaki))",
-    "author": "dtaniwaki",
-    "author_url": "https://github.com/dtaniwaki",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80802",
-    "pr_number": 80802,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.5"
-  },
-  "80804": {
-    "commit": "547617a523d9a1e6db9be68258303d5adb416e78",
-    "text": "\"kubeadm join\" fails if file-based discovery is too long, with a default timeout of 5 minutes.",
-    "markdown": "\"kubeadm join\" fails if file-based discovery is too long, with a default timeout of 5 minutes. ([#80804](https://github.com/kubernetes/kubernetes/pull/80804), [@olivierlemasle](https://github.com/olivierlemasle))",
-    "author": "olivierlemasle",
-    "author_url": "https://github.com/olivierlemasle",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80804",
-    "pr_number": 80804,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.5"
-  },
-  "80841": {
-    "commit": "9c22ff19663ffb43491152c41e2292325d021ef5",
-    "text": "enhance Azure cloud provider code to support both AAD and ADFS authentication.",
-    "markdown": "enhance Azure cloud provider code to support both AAD and ADFS authentication. ([#80841](https://github.com/kubernetes/kubernetes/pull/80841), [@rjaini](https://github.com/rjaini))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "rjaini",
-    "author_url": "https://github.com/rjaini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80841",
-    "pr_number": 80841,
-    "areas": ["cloudprovider"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.14.5"
-  }
-}
diff --git a/src/assets/release-notes-1.14.6.json b/src/assets/release-notes-1.14.6.json
deleted file mode 100644
index f3c50e39..00000000
--- a/src/assets/release-notes-1.14.6.json
+++ /dev/null
@@ -1,827 +0,0 @@
-{
-  "76463": {
-    "commit": "9dad928e6ba1d7cf8b67a9f401775d2c8b71a2d0",
-    "text": "Non nil DataSource entries on PVC's are now displayed as part of `describe pvc` output.",
-    "markdown": "Non nil DataSource entries on PVC's are now displayed as part of `describe pvc` output. ([#76463](https://github.com/kubernetes/kubernetes/pull/76463), [@j-griffith](https://github.com/j-griffith))",
-    "author": "j-griffith",
-    "author_url": "https://github.com/j-griffith",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76463",
-    "pr_number": 76463,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.6"
-  },
-  "77167": {
-    "commit": "c8c9d2e21ce9afe52ac8ace64f96a469891dc724",
-    "text": "Improvement in Kube-proxy. Kube-proxy waits for some duration for the node to be defined.",
-    "markdown": "Improvement in Kube-proxy. Kube-proxy waits for some duration for the node to be defined. ([#77167](https://github.com/kubernetes/kubernetes/pull/77167), [@paulsubrata55](https://github.com/paulsubrata55))",
-    "author": "paulsubrata55",
-    "author_url": "https://github.com/paulsubrata55",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77167",
-    "pr_number": 77167,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.6"
-  },
-  "78097": {
-    "commit": "cc270c138da473430d7243e1fbe4fbadf03cc49c",
-    "text": "Implement \"post-filter\" extension point for scheduling framework",
-    "markdown": "Implement \"post-filter\" extension point for scheduling framework ([#78097](https://github.com/kubernetes/kubernetes/pull/78097), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78097",
-    "pr_number": 78097,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.6"
-  },
-  "78775": {
-    "commit": "a7c81c663d4a59d0e4d9be96c394b7dad1a6cf76",
-    "text": "kube-proxy --cleanup will return the correct exit code if the cleanup was successful",
-    "markdown": "kube-proxy --cleanup will return the correct exit code if the cleanup was successful ([#78775](https://github.com/kubernetes/kubernetes/pull/78775), [@johscheuer](https://github.com/johscheuer))",
-    "author": "johscheuer",
-    "author_url": "https://github.com/johscheuer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78775",
-    "pr_number": 78775,
-    "kinds": ["cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.14.6"
-  },
-  "78793": {
-    "commit": "c63000ef81eac07508892e0aac9208b498ac9896",
-    "text": "When specifying `--(kube|system)-reserved-cgroup`, with `--cgroup-driver=systemd`, it is now possible to use the fully qualified cgroupfs name (i.e. `/test-cgroup.slice`).",
-    "markdown": "When specifying `--(kube|system)-reserved-cgroup`, with `--cgroup-driver=systemd`, it is now possible to use the fully qualified cgroupfs name (i.e. `/test-cgroup.slice`). ([#78793](https://github.com/kubernetes/kubernetes/pull/78793), [@mattjmcnaughton](https://github.com/mattjmcnaughton))",
-    "author": "mattjmcnaughton",
-    "author_url": "https://github.com/mattjmcnaughton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78793",
-    "pr_number": 78793,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.6"
-  },
-  "79033": {
-    "commit": "e6d427339593383df34ade066f7d571193686aae",
-    "text": "kubeadm: use the --pod-network-cidr flag to init or use the podSubnet field in the kubeadm config to pass a comma separated list of pod CIDRs.",
-    "markdown": "kubeadm: use the --pod-network-cidr flag to init or use the podSubnet field in the kubeadm config to pass a comma separated list of pod CIDRs. ([#79033](https://github.com/kubernetes/kubernetes/pull/79033), [@Arvinderpal](https://github.com/Arvinderpal))\n\n  Courtesy of SIG Cluster Lifecycle, SIG Network, and SIG Testing",
-    "author": "Arvinderpal",
-    "author_url": "https://github.com/Arvinderpal",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79033",
-    "pr_number": 79033,
-    "areas": ["kubeadm", "test"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "network", "testing"],
-    "feature": true,
-    "release_version": "1.14.6"
-  },
-  "79263": {
-    "commit": "c2286aa9bdf32af3661c272adb7707ff3439f2ff",
-    "text": "Remove DirectCodecFactory(replace with serializer.WithoutConversionCodecFactory), DirectEncoder(replace with runtime.WithVersionEncoder) and DirectDecoder(replace with runtime.WithoutVersionDecoder). action required",
-    "markdown": "Remove DirectCodecFactory(replace with serializer.WithoutConversionCodecFactory), DirectEncoder(replace with runtime.WithVersionEncoder) and DirectDecoder(replace with runtime.WithoutVersionDecoder). action required ([#79263](https://github.com/kubernetes/kubernetes/pull/79263), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG API Machinery",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79263",
-    "pr_number": 79263,
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "action_required": true,
-    "release_version": "1.14.6"
-  },
-  "79574": {
-    "commit": "1ff4525a29f1736fca19e10b49e42664b29010c2",
-    "text": "Update Azure SDK + go-autorest API versions",
-    "markdown": "Update Azure SDK + go-autorest API versions ([#79574](https://github.com/kubernetes/kubernetes/pull/79574), [@justaugustus](https://github.com/justaugustus))\n\n  Courtesy of SIG API Machinery, SIG CLI, SIG Cloud Provider, SIG Cluster Lifecycle, and SIG Node",
-    "author": "justaugustus",
-    "author_url": "https://github.com/justaugustus",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79574",
-    "pr_number": 79574,
-    "areas": [
-      "apiserver",
-      "cloudprovider",
-      "code-generation",
-      "dependency",
-      "kubectl",
-      "provider/azure"
-    ],
-    "kinds": ["cleanup", "feature"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "node"],
-    "feature": true,
-    "release_version": "1.14.6"
-  },
-  "79641": {
-    "commit": "2ad2795136be1edd0d1920e7b3e8c25e1e66f6a4",
-    "text": "If scheduler extender filtered a not found node, current scheduling round for this pod will just be skipped.",
-    "markdown": "If scheduler extender filtered a not found node, current scheduling round for this pod will just be skipped. ([#79641](https://github.com/kubernetes/kubernetes/pull/79641), [@yqwang-ms](https://github.com/yqwang-ms))",
-    "author": "yqwang-ms",
-    "author_url": "https://github.com/yqwang-ms",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79641",
-    "pr_number": 79641,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.6"
-  },
-  "79722": {
-    "commit": "133f37814c0ad5d140f9fcb7c53e467cebeed65d",
-    "text": "etcd Docker image can be run as non-root",
-    "markdown": "etcd Docker image can be run as non-root ([#79722](https://github.com/kubernetes/kubernetes/pull/79722), [@randomvariable](https://github.com/randomvariable))",
-    "author": "randomvariable",
-    "author_url": "https://github.com/randomvariable",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79722",
-    "pr_number": 79722,
-    "areas": ["security"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "release"],
-    "duplicate": true,
-    "release_version": "1.14.6"
-  },
-  "79804": {
-    "commit": "a56e86a405a414f83e5bb1d4e84e37d80af543b7",
-    "text": "Extender bind should respect IsInterested",
-    "markdown": "Extender bind should respect IsInterested ([#79804](https://github.com/kubernetes/kubernetes/pull/79804), [@yqwang-ms](https://github.com/yqwang-ms))",
-    "author": "yqwang-ms",
-    "author_url": "https://github.com/yqwang-ms",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79804",
-    "pr_number": 79804,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.6"
-  },
-  "80231": {
-    "commit": "c981c65c9013306d9a65aedbd30479b932e427b2",
-    "text": "The `AdmissionReview` API sent to and received from admission webhooks has been promoted to `admission.k8s.io/v1`. Webhooks can specify a preference for receiving `v1` AdmissionReview objects with `admissionReviewVersions: [\"v1\",\"v1beta1\"]`, and must respond with an API object in the same `apiVersion` they are sent. When webhooks use `admission.k8s.io/v1`, the following additional validation is performed on their responses:\n- `response.patch` and `response.patchType` are not permitted from validating admission webhooks\n- `apiVersion: \"admission.k8s.io/v1\"` is required\n- `kind: \"AdmissionReview\"` is required\n- `response.uid: \"\u003cvalue of request.uid\u003e\"` is required\n- `response.patchType: \"JSONPatch\"` is required (if `response.patch` is set)",
-    "markdown": "The `AdmissionReview` API sent to and received from admission webhooks has been promoted to `admission.k8s.io/v1`. Webhooks can specify a preference for receiving `v1` AdmissionReview objects with `admissionReviewVersions: [\"v1\",\"v1beta1\"]`, and must respond with an API object in the same `apiVersion` they are sent. When webhooks use `admission.k8s.io/v1`, the following additional validation is performed on their responses:\n  - `response.patch` and `response.patchType` are not permitted from validating admission webhooks\n  - `apiVersion: \"admission.k8s.io/v1\"` is required\n  - `kind: \"AdmissionReview\"` is required\n  - `response.uid: \"\u003cvalue of request.uid\u003e\"` is required\n  - `response.patchType: \"JSONPatch\"` is required (if `response.patch` is set) ([#80231](https://github.com/kubernetes/kubernetes/pull/80231), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG CLI, SIG Node, and SIG Testing",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80231",
-    "pr_number": 80231,
-    "areas": ["admission-control", "apiserver", "kubectl", "kubelet", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "auth", "cli", "node", "testing"],
-    "feature": true,
-    "release_version": "1.14.6"
-  },
-  "80342": {
-    "commit": "273e9262bb1de2e31a57065ea082dee8585e2ed1",
-    "text": "scheduler.alpha.kubernetes.io/critical-pod annotation is removed. Pod priority (spec.priorityClassName) should be used instead to mark pods as critical. Action required!",
-    "markdown": "scheduler.alpha.kubernetes.io/critical-pod annotation is removed. Pod priority (spec.priorityClassName) should be used instead to mark pods as critical. Action required! ([#80342](https://github.com/kubernetes/kubernetes/pull/80342), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Cluster Lifecycle, SIG Instrumentation, SIG Node, SIG Scheduling, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80342",
-    "pr_number": 80342,
-    "areas": ["apiserver", "kubelet", "provider/gcp", "test"],
-    "kinds": ["cleanup"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "cluster-lifecycle",
-      "instrumentation",
-      "node",
-      "scheduling",
-      "testing"
-    ],
-    "action_required": true,
-    "release_version": "1.14.6"
-  },
-  "80369": {
-    "commit": "1fcd3d6baced5aeb3f958d0819afc71db96bcc23",
-    "text": "Errors from pod volume set up are now propagated as pod events.",
-    "markdown": "Errors from pod volume set up are now propagated as pod events. ([#80369](https://github.com/kubernetes/kubernetes/pull/80369), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80369",
-    "pr_number": 80369,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.6"
-  },
-  "80376": {
-    "commit": "f6a70ef27163d1171412b6f97049968896b8d1f9",
-    "text": "Removed cadvisor metric labels `pod_name` and `container_name` to match instrumentation guidelines.\n\nany Prometheus queries that match `pod_name` and `container_name` labels (e.g. cadvisor or kubelet probe metrics) must be updated to use `pod` and `container` instead.",
-    "markdown": "Removed cadvisor metric labels `pod_name` and `container_name` to match instrumentation guidelines.\n  \n  any Prometheus queries that match `pod_name` and `container_name` labels (e.g. cadvisor or kubelet probe metrics) must be updated to use `pod` and `container` instead. ([#80376](https://github.com/kubernetes/kubernetes/pull/80376), [@ehashman](https://github.com/ehashman))\n\n  Courtesy of SIG Instrumentation, SIG Node, and SIG Testing",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/28d0f41a30b32bceb01e0d4ae338f91ede195005/keps/sig-instrumentation/20181106-kubernetes-metrics-overhaul.md#cadvisor-instrumentation-changes",
-        "type": "KEP"
-      }
-    ],
-    "author": "ehashman",
-    "author_url": "https://github.com/ehashman",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80376",
-    "pr_number": 80376,
-    "areas": ["kubelet", "test"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["instrumentation", "node", "testing"],
-    "action_required": true,
-    "release_version": "1.14.6"
-  },
-  "80491": {
-    "commit": "806ced34a42111eff1bfafb46b4838db0a9a3bfc",
-    "text": "Kubelet considers all static pods as critical. Static pods pass kubelet admission even if a node does not have enough resources. Users must ensure that they account for resources when creating static pods.",
-    "markdown": "Kubelet considers all static pods as critical. Static pods pass kubelet admission even if a node does not have enough resources. Users must ensure that they account for resources when creating static pods. ([#80491](https://github.com/kubernetes/kubernetes/pull/80491), [@hpandeycodeit](https://github.com/hpandeycodeit))",
-    "author": "hpandeycodeit",
-    "author_url": "https://github.com/hpandeycodeit",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80491",
-    "pr_number": 80491,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.14.6"
-  },
-  "80699": {
-    "commit": "182ae7b9a47f14f55c3fed5f8f948ace647263e9",
-    "text": "Add a `Patch` method to `ScaleInterface`",
-    "markdown": "Add a `Patch` method to `ScaleInterface` ([#80699](https://github.com/kubernetes/kubernetes/pull/80699), [@knight42](https://github.com/knight42))\n\n  Courtesy of SIG API Machinery",
-    "author": "knight42",
-    "author_url": "https://github.com/knight42",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80699",
-    "pr_number": 80699,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.6"
-  },
-  "80730": {
-    "commit": "9743d7f27081660b65cb85f17d46829c11d56cee",
-    "text": "TBD",
-    "markdown": "TBD ([#80730](https://github.com/kubernetes/kubernetes/pull/80730), [@jennybuckley](https://github.com/jennybuckley))\n\n  Courtesy of SIG API Machinery, SIG Cloud Provider, and SIG Testing",
-    "author": "jennybuckley",
-    "author_url": "https://github.com/jennybuckley",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80730",
-    "pr_number": 80730,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "cloud-provider", "testing"],
-    "feature": true,
-    "release_version": "1.14.6"
-  },
-  "80764": {
-    "commit": "56d7920e8fdefe9505b4e6ce4798619940ca0a28",
-    "text": "Determine system model to get credentials for windows nodes.",
-    "markdown": "Determine system model to get credentials for windows nodes. ([#80764](https://github.com/kubernetes/kubernetes/pull/80764), [@liyanhui1228](https://github.com/liyanhui1228))",
-    "author": "liyanhui1228",
-    "author_url": "https://github.com/liyanhui1228",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80764",
-    "pr_number": 80764,
-    "kinds": ["bug"],
-    "release_version": "1.14.6"
-  },
-  "80825": {
-    "commit": "2f91fcc3fd85470128981b2051152f560dca4cf0",
-    "text": "Add scheduling support for RuntimeClasses. RuntimeClasses can now specify nodeSelector constraints \u0026 tolerations, which are merged into the PodSpec for pods using that RuntimeClass.",
-    "markdown": "Add scheduling support for RuntimeClasses. RuntimeClasses can now specify nodeSelector constraints \u0026 tolerations, which are merged into the PodSpec for pods using that RuntimeClass. ([#80825](https://github.com/kubernetes/kubernetes/pull/80825), [@tallclair](https://github.com/tallclair))",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/runtime-class-scheduling.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80825",
-    "pr_number": 80825,
-    "kinds": ["api-change"],
-    "sigs": ["node"],
-    "release_version": "1.14.6"
-  },
-  "80837": {
-    "commit": "16d9a659da541fc38110c112c6ae20a1056baf37",
-    "text": "Azure disks of shared kind will no longer fail if they do not contain skuname or \n    storageaccounttype.",
-    "markdown": "Azure disks of shared kind will no longer fail if they do not contain skuname or \n      storageaccounttype. ([#80837](https://github.com/kubernetes/kubernetes/pull/80837), [@rmweir](https://github.com/rmweir))",
-    "author": "rmweir",
-    "author_url": "https://github.com/rmweir",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80837",
-    "pr_number": 80837,
-    "areas": ["provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.6"
-  },
-  "80861": {
-    "commit": "231a6a7850d2bd4eb1ae85f87dbc9ca14c373589",
-    "text": "kube-addon-manager has been updated to v9.0.2 to fix a bug in leader election (https://github.com/kubernetes/kubernetes/pull/80575)",
-    "markdown": "kube-addon-manager has been updated to v9.0.2 to fix a bug in leader election (https://github.com/kubernetes/kubernetes/pull/80575) ([#80861](https://github.com/kubernetes/kubernetes/pull/80861), [@mborsz](https://github.com/mborsz))",
-    "author": "mborsz",
-    "author_url": "https://github.com/mborsz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80861",
-    "pr_number": 80861,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.6"
-  },
-  "80862": {
-    "commit": "564eff757cdd26a1322458876287298b8434b78b",
-    "text": "kubeadm: treat non-fatal errors as warnings when doing reset",
-    "markdown": "kubeadm: treat non-fatal errors as warnings when doing reset ([#80862](https://github.com/kubernetes/kubernetes/pull/80862), [@drpaneas](https://github.com/drpaneas))",
-    "author": "drpaneas",
-    "author_url": "https://github.com/drpaneas",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80862",
-    "pr_number": 80862,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.6"
-  },
-  "80867": {
-    "commit": "44f3aafc1a8ce9c1b5972cbfa12aa9b583fdfe7b",
-    "text": "This PR fixes an error when using external etcd but storing etcd certificates in the same folder and with the same name used by kubeadm for local etcd certificates; for an older version of kubeadm, the workaround is to avoid file name used by kubeadm for local etcd.",
-    "markdown": "This PR fixes an error when using external etcd but storing etcd certificates in the same folder and with the same name used by kubeadm for local etcd certificates; for an older version of kubeadm, the workaround is to avoid file name used by kubeadm for local etcd. ([#80867](https://github.com/kubernetes/kubernetes/pull/80867), [@fabriziopandini](https://github.com/fabriziopandini))",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80867",
-    "pr_number": 80867,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.6"
-  },
-  "80901": {
-    "commit": "1906650d9284ffebe97cb0abd17f8fbdb5c63b93",
-    "text": "Use a named array instead of a score array in normalizing-score phase.",
-    "markdown": "Use a named array instead of a score array in normalizing-score phase. ([#80901](https://github.com/kubernetes/kubernetes/pull/80901), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80901",
-    "pr_number": 80901,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.6"
-  },
-  "80905": {
-    "commit": "c08ee9d51b31c5dcda54e291694feb1cd747b7eb",
-    "text": "Implement a new feature that allows applying kustomize patches to static pod manifests generated by kubeadm.",
-    "markdown": "Implement a new feature that allows applying kustomize patches to static pod manifests generated by kubeadm. ([#80905](https://github.com/kubernetes/kubernetes/pull/80905), [@fabriziopandini](https://github.com/fabriziopandini))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-cluster-lifecycle/kubeadm/20190722-Advanced-configurations-with-kubeadm-(Kustomize).md",
-        "type": "KEP"
-      }
-    ],
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80905",
-    "pr_number": 80905,
-    "areas": ["dependency", "kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.6"
-  },
-  "80923": {
-    "commit": "5c16a784c694734060aeb7f4f650d0cf11216b9d",
-    "text": "Reduces GCE PD Node Attach Limits by 1 since the node boot disk is considered an attachable disk",
-    "markdown": "Reduces GCE PD Node Attach Limits by 1 since the node boot disk is considered an attachable disk ([#80923](https://github.com/kubernetes/kubernetes/pull/80923), [@davidz627](https://github.com/davidz627))",
-    "author": "davidz627",
-    "author_url": "https://github.com/davidz627",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80923",
-    "pr_number": 80923,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.6"
-  },
-  "80942": {
-    "commit": "12a085f9176cfd7fa4514922f6481a3d1787b207",
-    "text": "Fix a bug in the IPVS proxier where virtual servers are not cleaned up even though the corresponding Service object was deleted.",
-    "markdown": "Fix a bug in the IPVS proxier where virtual servers are not cleaned up even though the corresponding Service object was deleted. ([#80942](https://github.com/kubernetes/kubernetes/pull/80942), [@gongguan](https://github.com/gongguan))",
-    "author": "gongguan",
-    "author_url": "https://github.com/gongguan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80942",
-    "pr_number": 80942,
-    "areas": ["ipvs"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.6"
-  },
-  "80945": {
-    "commit": "6049253aae49f87985a59ba45bedded94023b093",
-    "text": "Fixes validation of VolumeAttachment API objects created with inline volume sources.",
-    "markdown": "Fixes validation of VolumeAttachment API objects created with inline volume sources. ([#80945](https://github.com/kubernetes/kubernetes/pull/80945), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80945",
-    "pr_number": 80945,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.6"
-  },
-  "80951": {
-    "commit": "74c0cc27902548243d8a863c9e2c6345a8e6b548",
-    "text": "kubeadm: enable secure serving for the kube-scheduler",
-    "markdown": "kubeadm: enable secure serving for the kube-scheduler ([#80951](https://github.com/kubernetes/kubernetes/pull/80951), [@neolit123](https://github.com/neolit123))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80951",
-    "pr_number": 80951,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.6"
-  },
-  "80966": {
-    "commit": "da30406a9d993f437c5d971f33f44a739312a914",
-    "text": "kubeadm: fix \"certificate-authority\" files not being pre-loaded when using file discovery",
-    "markdown": "kubeadm: fix \"certificate-authority\" files not being pre-loaded when using file discovery ([#80966](https://github.com/kubernetes/kubernetes/pull/80966), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80966",
-    "pr_number": 80966,
-    "areas": ["kubeadm"],
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.6"
-  },
-  "80978": {
-    "commit": "ef8869466c803fbbe44a411a8a2e3e37c7f1d1d2",
-    "text": "API: the metadata.selfLink field is deprecated in individual and list objects. It will no longer be returned starting in v1.20, and the field will be removed entirely in v1.21.",
-    "markdown": "API: the metadata.selfLink field is deprecated in individual and list objects. It will no longer be returned starting in v1.20, and the field will be removed entirely in v1.21. ([#80978](https://github.com/kubernetes/kubernetes/pull/80978), [@wojtek-t](https://github.com/wojtek-t))",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80978",
-    "pr_number": 80978,
-    "areas": ["apiserver", "code-generation", "kubelet"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "apps", "node"],
-    "duplicate": true,
-    "release_version": "1.14.6"
-  },
-  "81015": {
-    "commit": "7a35f8c92dd5125924873fe4db8fd82b8889c99a",
-    "text": "Return error when the scoring plugin returns score out of range [0, 100].",
-    "markdown": "Return error when the scoring plugin returns score out of range [0, 100]. ([#81015](https://github.com/kubernetes/kubernetes/pull/81015), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-scheduling/20180409-scheduling-framework.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81015",
-    "pr_number": 81015,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.6"
-  },
-  "81029": {
-    "commit": "62f1c4065b4c4b1615833245384e8de81bd530d9",
-    "text": "cloud-controller-manager binaries and docker images are no longer shipped with kubernetes releases.",
-    "markdown": "cloud-controller-manager binaries and docker images are no longer shipped with kubernetes releases. ([#81029](https://github.com/kubernetes/kubernetes/pull/81029), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81029",
-    "pr_number": 81029,
-    "kinds": ["cleanup"],
-    "sigs": ["cloud-provider", "release"],
-    "duplicate": true,
-    "release_version": "1.14.6"
-  },
-  "81054": {
-    "commit": "8527c74e40f0b3b8cf5f5fe80e4fff607a147c1a",
-    "text": "Add a azure cloud configuration `LoadBalancerName` and `LoadBalancerResourceGroup` to allow the corresponding customizations of azure load balancer.",
-    "markdown": "Add a azure cloud configuration `LoadBalancerName` and `LoadBalancerResourceGroup` to allow the corresponding customizations of azure load balancer. ([#81054](https://github.com/kubernetes/kubernetes/pull/81054), [@nilo19](https://github.com/nilo19))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81054",
-    "pr_number": 81054,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.14.6"
-  },
-  "81094": {
-    "commit": "9690201481415a548dff6866d836c2ad2a87d585",
-    "text": "Deprecate the `--cloud-provider-gce-lb-src-cidrs` flag in the kube-apiserver. This flag will be removed once the GCE Cloud Provider is removed from kube-apiserver.",
-    "markdown": "Deprecate the `--cloud-provider-gce-lb-src-cidrs` flag in the kube-apiserver. This flag will be removed once the GCE Cloud Provider is removed from kube-apiserver. ([#81094](https://github.com/kubernetes/kubernetes/pull/81094), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81094",
-    "pr_number": 81094,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "network"],
-    "duplicate": true,
-    "release_version": "1.14.6"
-  },
-  "81096": {
-    "commit": "3030a9d92750173ed72ea9cd37286bf6a41c2dcb",
-    "text": "Fix a bug that pretty printer marshals empty byte or uint8 slice as null",
-    "markdown": "Fix a bug that pretty printer marshals empty byte or uint8 slice as null ([#81096](https://github.com/kubernetes/kubernetes/pull/81096), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81096",
-    "pr_number": 81096,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle"],
-    "duplicate": true,
-    "release_version": "1.14.6"
-  },
-  "81106": {
-    "commit": "18b6ff3d65a77b1b257af6e1128b322c7de0031b",
-    "text": "Update the GCE windows node image to include hot fixes since July.",
-    "markdown": "Update the GCE windows node image to include hot fixes since July. ([#81106](https://github.com/kubernetes/kubernetes/pull/81106), [@YangLu1031](https://github.com/YangLu1031))",
-    "author": "YangLu1031",
-    "author_url": "https://github.com/YangLu1031",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81106",
-    "pr_number": 81106,
-    "kinds": ["cleanup", "flake"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.6"
-  },
-  "81152": {
-    "commit": "f4e39afea082afac7bf6f6936c34e012b7cb3c25",
-    "text": "kube-apiserver: the `--basic-auth-file` flag and authentication mode is deprecated and will be removed in a future release. It is not recommended for production environments.",
-    "markdown": "kube-apiserver: the `--basic-auth-file` flag and authentication mode is deprecated and will be removed in a future release. It is not recommended for production environments. ([#81152](https://github.com/kubernetes/kubernetes/pull/81152), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81152",
-    "pr_number": 81152,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery", "auth"],
-    "duplicate": true,
-    "release_version": "1.14.6"
-  },
-  "81174": {
-    "commit": "c0997e8dc1dfe5d804917e1d0c7d97d4c6fdde9e",
-    "text": "Fix kubelet NodeLease potential performance issues. Kubelet now will try to update lease using cached one instead of get from API Server every time.",
-    "markdown": "Fix kubelet NodeLease potential performance issues. Kubelet now will try to update lease using cached one instead of get from API Server every time. ([#81174](https://github.com/kubernetes/kubernetes/pull/81174), [@answer1991](https://github.com/answer1991))",
-    "author": "answer1991",
-    "author_url": "https://github.com/answer1991",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81174",
-    "pr_number": 81174,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.6"
-  },
-  "81208": {
-    "commit": "a4c5a578004b71e41fc05c167ea96b6016046ce6",
-    "text": "switch to VM Update call in attach/detach disk operation, original CreateOrUpdate call may lead to orphaned VMs or blocked resources",
-    "markdown": "switch to VM Update call in attach/detach disk operation, original CreateOrUpdate call may lead to orphaned VMs or blocked resources ([#81208](https://github.com/kubernetes/kubernetes/pull/81208), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81208",
-    "pr_number": 81208,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.6"
-  },
-  "81210": {
-    "commit": "406904269d5a757bedd57be4d49b83d5f57a5847",
-    "text": "added an new Prometheus counter metric \"sync_proxy_rules_iptables_restore_failures_total\" for kube-proxy iptables-restore failures (both ipvs and iptables modes)",
-    "markdown": "added an new Prometheus counter metric \"sync_proxy_rules_iptables_restore_failures_total\" for kube-proxy iptables-restore failures (both ipvs and iptables modes) ([#81210](https://github.com/kubernetes/kubernetes/pull/81210), [@figo](https://github.com/figo))\n\n  Courtesy of SIG Network",
-    "author": "figo",
-    "author_url": "https://github.com/figo",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81210",
-    "pr_number": 81210,
-    "areas": ["ipvs"],
-    "kinds": ["feature"],
-    "sigs": ["network"],
-    "feature": true,
-    "release_version": "1.14.6"
-  },
-  "81212": {
-    "commit": "2c455e0ac0aceeee74281e0f3fb199109921bb49",
-    "text": "Improves validation errors for custom resources",
-    "markdown": "Improves validation errors for custom resources ([#81212](https://github.com/kubernetes/kubernetes/pull/81212), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81212",
-    "pr_number": 81212,
-    "areas": ["custom-resources", "dependency"],
-    "kinds": ["api-change", "cleanup", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.6"
-  },
-  "81213": {
-    "commit": "a12b64802959b4d826137e7eb1512f0058109bd3",
-    "text": "Add a service annotation `service.beta.kubernetes.io/azure-pip-name` to specify the public IP name for Azure load balancer.",
-    "markdown": "Add a service annotation `service.beta.kubernetes.io/azure-pip-name` to specify the public IP name for Azure load balancer. ([#81213](https://github.com/kubernetes/kubernetes/pull/81213), [@nilo19](https://github.com/nilo19))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81213",
-    "pr_number": 81213,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.14.6"
-  },
-  "81215": {
-    "commit": "d23b2c7d83fc7b888a7d6dfd4ba940e8d7c06cb6",
-    "text": "remove iSCSI volume storage cleartext secrets in logs",
-    "markdown": "remove iSCSI volume storage cleartext secrets in logs ([#81215](https://github.com/kubernetes/kubernetes/pull/81215), [@zouyee](https://github.com/zouyee))",
-    "author": "zouyee",
-    "author_url": "https://github.com/zouyee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81215",
-    "pr_number": 81215,
-    "kinds": ["bug", "flake"],
-    "sigs": ["storage"],
-    "release_version": "1.14.6"
-  },
-  "81217": {
-    "commit": "ca6290538b91ebd6fa65408a5d4030efa8c15918",
-    "text": "kubeadm: the permissions of generated CSR files are changed from 0644 to 0600",
-    "markdown": "kubeadm: the permissions of generated CSR files are changed from 0644 to 0600 ([#81217](https://github.com/kubernetes/kubernetes/pull/81217), [@SataQiu](https://github.com/SataQiu))",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81217",
-    "pr_number": 81217,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.6"
-  },
-  "81219": {
-    "commit": "06fb339e31b127bf7d506a656b02188f303f7ddd",
-    "text": "hyperkube will drop support for cloud-controller-manager in a future release",
-    "markdown": "hyperkube will drop support for cloud-controller-manager in a future release ([#81219](https://github.com/kubernetes/kubernetes/pull/81219), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81219",
-    "pr_number": 81219,
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cloud-provider", "release"],
-    "duplicate": true,
-    "release_version": "1.14.6"
-  },
-  "81244": {
-    "commit": "3f0a486cbf43abe113923ed6947b8d338574ad43",
-    "text": "Resolves a transient 404 response to custom resource requests during server startup",
-    "markdown": "Resolves a transient 404 response to custom resource requests during server startup ([#81244](https://github.com/kubernetes/kubernetes/pull/81244), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81244",
-    "pr_number": 81244,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.6"
-  },
-  "81262": {
-    "commit": "b5d33fa9bd8ff6ecfee2df33ccd00c21a09dc5a1",
-    "text": "fix azure load balancer update dns label issue",
-    "markdown": "fix azure load balancer update dns label issue ([#81262](https://github.com/kubernetes/kubernetes/pull/81262), [@levimm](https://github.com/levimm))",
-    "author": "levimm",
-    "author_url": "https://github.com/levimm",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81262",
-    "pr_number": 81262,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.6"
-  },
-  "81279": {
-    "commit": "54b28ab60c66746f314fe2ed109c46893f37a71e",
-    "text": "Fix Azure client requests stuck issues on http.StatusTooManyRequests (HTTP Code 429).",
-    "markdown": "Fix Azure client requests stuck issues on http.StatusTooManyRequests (HTTP Code 429). ([#81279](https://github.com/kubernetes/kubernetes/pull/81279), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81279",
-    "pr_number": 81279,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.6"
-  },
-  "81282": {
-    "commit": "ae457448d6963ffbb6427afc8c87598f89fadf71",
-    "text": "Fix conflicted cache when the requests are canceled by other Azure operations.",
-    "markdown": "Fix conflicted cache when the requests are canceled by other Azure operations. ([#81282](https://github.com/kubernetes/kubernetes/pull/81282), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81282",
-    "pr_number": 81282,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.6"
-  },
-  "81306": {
-    "commit": "de3aed802338437f189313dcd6b17ddd9bb15e95",
-    "text": "Scheduler should terminate when it looses leader lock.",
-    "markdown": "Scheduler should terminate when it looses leader lock. ([#81306](https://github.com/kubernetes/kubernetes/pull/81306), [@ravisantoshgudimetla](https://github.com/ravisantoshgudimetla))",
-    "author": "ravisantoshgudimetla",
-    "author_url": "https://github.com/ravisantoshgudimetla",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81306",
-    "pr_number": 81306,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.6"
-  },
-  "81330": {
-    "commit": "4441f1d9c3e94d9a3d93b4f184a591cab02a5245",
-    "text": "Fixes CVE-2019-11250: client-go header logging (at verbosity levels \u003e= 7) now masks `Authorization` header contents",
-    "markdown": "Fixes CVE-2019-11250: client-go header logging (at verbosity levels \u003e= 7) now masks `Authorization` header contents ([#81330](https://github.com/kubernetes/kubernetes/pull/81330), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81330",
-    "pr_number": 81330,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.6"
-  },
-  "81337": {
-    "commit": "2974adff27619bb27a27a78bd61de772a2d65bef",
-    "text": "Add instruction to setup \"Application Default Credentials\" to run GCE Windows e2e tests locally.",
-    "markdown": "Add instruction to setup \"Application Default Credentials\" to run GCE Windows e2e tests locally. ([#81337](https://github.com/kubernetes/kubernetes/pull/81337), [@YangLu1031](https://github.com/YangLu1031))",
-    "author": "YangLu1031",
-    "author_url": "https://github.com/YangLu1031",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81337",
-    "pr_number": 81337,
-    "kinds": ["documentation"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.6"
-  },
-  "81377": {
-    "commit": "3645041af5ec0a4f55a3e9eaa4cd3aec402cca96",
-    "text": "kubelet now exports an \"kubelet_evictions\" metric that counts the number of pod evictions carried out by the kubelet to reclaim resources",
-    "markdown": "kubelet now exports an \"kubelet_evictions\" metric that counts the number of pod evictions carried out by the kubelet to reclaim resources ([#81377](https://github.com/kubernetes/kubernetes/pull/81377), [@sjenning](https://github.com/sjenning))\n\n  Courtesy of SIG Instrumentation, and SIG Node",
-    "author": "sjenning",
-    "author_url": "https://github.com/sjenning",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81377",
-    "pr_number": 81377,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["instrumentation", "node"],
-    "feature": true,
-    "release_version": "1.14.6"
-  },
-  "81385": {
-    "commit": "9e60bedebed6be97a91dda82451e8f82d66c34d4",
-    "text": "kubeadm: use etcd's /health endpoint for a HTTP liveness probe on localhost instead of having a custom health check using etcdctl",
-    "markdown": "kubeadm: use etcd's /health endpoint for a HTTP liveness probe on localhost instead of having a custom health check using etcdctl ([#81385](https://github.com/kubernetes/kubernetes/pull/81385), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81385",
-    "pr_number": 81385,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.6"
-  },
-  "81390": {
-    "commit": "3526a6162e7165966c3adb811ef82d20fabe17c8",
-    "text": "Update to use go 1.12.8",
-    "markdown": "Update to use go 1.12.8 ([#81390](https://github.com/kubernetes/kubernetes/pull/81390), [@cblecker](https://github.com/cblecker))",
-    "author": "cblecker",
-    "author_url": "https://github.com/cblecker",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81390",
-    "pr_number": 81390,
-    "areas": ["cloudprovider", "test"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "release", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.6"
-  },
-  "81394": {
-    "commit": "1f6cb3cb9def97320a5412dcbea1661edd95c29e",
-    "text": "Update golang/x/net dependency to bring in fixes for CVE-2019-9512, CVE-2019-9514",
-    "markdown": "Update golang/x/net dependency to bring in fixes for CVE-2019-9512, CVE-2019-9514 ([#81394](https://github.com/kubernetes/kubernetes/pull/81394), [@cblecker](https://github.com/cblecker))",
-    "author": "cblecker",
-    "author_url": "https://github.com/cblecker",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81394",
-    "pr_number": 81394,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "node", "release"],
-    "duplicate": true,
-    "release_version": "1.14.6"
-  },
-  "81489": {
-    "commit": "e283836fb2ced2c1d918eef43c837f27c9381fc9",
-    "text": "Update to use go 1.12.9",
-    "markdown": "Update to use go 1.12.9 ([#81489](https://github.com/kubernetes/kubernetes/pull/81489), [@BenTheElder](https://github.com/BenTheElder))",
-    "author": "BenTheElder",
-    "author_url": "https://github.com/BenTheElder",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81489",
-    "pr_number": 81489,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["release", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.6"
-  }
-}
diff --git a/src/assets/release-notes-1.14.7.json b/src/assets/release-notes-1.14.7.json
deleted file mode 100644
index 6db8e3a4..00000000
--- a/src/assets/release-notes-1.14.7.json
+++ /dev/null
@@ -1,1679 +0,0 @@
-{
-  "53185": {
-    "commit": "315dcca3417e6224294322b1073b5e9d03453f26",
-    "text": "populate object name for admission attributes when CREATE",
-    "markdown": "populate object name for admission attributes when CREATE ([#53185](https://github.com/kubernetes/kubernetes/pull/53185), [@dixudx](https://github.com/dixudx))\n\n  Courtesy of SIG API Machinery, SIG Auth, and SIG Testing",
-    "author": "dixudx",
-    "author_url": "https://github.com/dixudx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/53185",
-    "pr_number": 53185,
-    "areas": ["admission-control", "apiserver", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "auth", "testing"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "57741": {
-    "commit": "53f4e0fa26c25d7d4eed8fa7f0b3c363ce33f4c8",
-    "text": "Omit nil or empty field when calculating container hash value to avoid hash changed. For a new field with a non-nil default value in the container spec, the hash would still get changed.",
-    "markdown": "Omit nil or empty field when calculating container hash value to avoid hash changed. For a new field with a non-nil default value in the container spec, the hash would still get changed. ([#57741](https://github.com/kubernetes/kubernetes/pull/57741), [@dixudx](https://github.com/dixudx))",
-    "author": "dixudx",
-    "author_url": "https://github.com/dixudx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/57741",
-    "pr_number": 57741,
-    "areas": ["kubelet"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.7"
-  },
-  "59484": {
-    "commit": "59d29b67d94b3951320c1ee20481bb97425305d0",
-    "text": "Ephemeral containers have been added in alpha. These temporary containers can be added to running pods for purposes such as debugging, similar to how `kubectl exec` runs an process in an existing container. Also like `kubectl exec`, no resources are reserved for ephemeral containers and they are not restarted when they exit.\n\nNote that container namespace targeting is not yet implemented, so [process namespace sharing](https://kubernetes.io/docs/tasks/configure-pod-container/share-process-namespace/) must be enabled to view process from other containers in the pod.",
-    "markdown": "Ephemeral containers have been added in alpha. These temporary containers can be added to running pods for purposes such as debugging, similar to how `kubectl exec` runs an process in an existing container. Also like `kubectl exec`, no resources are reserved for ephemeral containers and they are not restarted when they exit.\n  \n  Note that container namespace targeting is not yet implemented, so [process namespace sharing](https://kubernetes.io/docs/tasks/configure-pod-container/share-process-namespace/) must be enabled to view process from other containers in the pod. ([#59484](https://github.com/kubernetes/kubernetes/pull/59484), [@verb](https://github.com/verb))\n\n  Courtesy of SIG Node",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://git.k8s.io/enhancements/keps/sig-node/20190212-ephemeral-containers.md",
-        "type": "external"
-      }
-    ],
-    "author": "verb",
-    "author_url": "https://github.com/verb",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/59484",
-    "pr_number": 59484,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "64996": {
-    "commit": "d951d19ec4d8ec829e83d146200ea91b664b6fa6",
-    "text": "Property `conditions` in `apiextensions.v1beta1.CustomResourceDefinitionStatus` and `apiextensions.v1.CustomResourceDefinitionStatus` is now optional instead of required.",
-    "markdown": "Property `conditions` in `apiextensions.v1beta1.CustomResourceDefinitionStatus` and `apiextensions.v1.CustomResourceDefinitionStatus` is now optional instead of required. ([#64996](https://github.com/kubernetes/kubernetes/pull/64996), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/64996",
-    "pr_number": 64996,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.7"
-  },
-  "69655": {
-    "commit": "bcb464db7ba2478429f4deff2c682f4efe46855e",
-    "text": "When the status of a CRD Condition changes, it's corresponding `LastTransitionTime` is now updated.",
-    "markdown": "When the status of a CRD Condition changes, it's corresponding `LastTransitionTime` is now updated. ([#69655](https://github.com/kubernetes/kubernetes/pull/69655), [@CaoShuFeng](https://github.com/CaoShuFeng))",
-    "author": "CaoShuFeng",
-    "author_url": "https://github.com/CaoShuFeng",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/69655",
-    "pr_number": 69655,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.7"
-  },
-  "71757": {
-    "commit": "bc1d8c6d61078269540f807248a455cba09e672c",
-    "text": "Implement request cancellation in token transport to get rid of warnings when the HTTP client tries to cancel a request.",
-    "markdown": "Implement request cancellation in token transport to get rid of warnings when the HTTP client tries to cancel a request. ([#71757](https://github.com/kubernetes/kubernetes/pull/71757), [@mikedanese](https://github.com/mikedanese))",
-    "author": "mikedanese",
-    "author_url": "https://github.com/mikedanese",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/71757",
-    "pr_number": 71757,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "73405": {
-    "commit": "efaacf786a3d04a9ee3fdc167fc2162221d95dfd",
-    "text": "Add status condition to namespace resource",
-    "markdown": "Add status condition to namespace resource ([#73405](https://github.com/kubernetes/kubernetes/pull/73405), [@wozniakjan](https://github.com/wozniakjan))\n\n  Courtesy of SIG API Machinery, SIG Apps, and SIG Architecture",
-    "author": "wozniakjan",
-    "author_url": "https://github.com/wozniakjan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73405",
-    "pr_number": 73405,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "architecture"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "77354": {
-    "commit": "ab162cd28c332d0ecfb4f918d5f91e9e57acdb61",
-    "text": "Server-side apply will now use the openapi provided in the CRD validation field to help figure out how to correctly merge objects and update ownership.",
-    "markdown": "Server-side apply will now use the openapi provided in the CRD validation field to help figure out how to correctly merge objects and update ownership. ([#77354](https://github.com/kubernetes/kubernetes/pull/77354), [@jennybuckley](https://github.com/jennybuckley))\n\n  Courtesy of SIG API Machinery, SIG CLI, SIG Cloud Provider, SIG Cluster Lifecycle, and SIG Testing",
-    "author": "jennybuckley",
-    "author_url": "https://github.com/jennybuckley",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77354",
-    "pr_number": 77354,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "testing"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "77688": {
-    "commit": "668bf42d11dd61a93e606ba0a1c9b9b6acbed2d3",
-    "text": "Updates the requestedToCapacityRatioArguments to add resources parameter that allows the users to specify the resource name along with weights for each resource to score nodes based on the request to capacity ratio.",
-    "markdown": "Updates the requestedToCapacityRatioArguments to add resources parameter that allows the users to specify the resource name along with weights for each resource to score nodes based on the request to capacity ratio. ([#77688](https://github.com/kubernetes/kubernetes/pull/77688), [@sudeshsh](https://github.com/sudeshsh))\n\n  Courtesy of SIG Scheduling",
-    "author": "sudeshsh",
-    "author_url": "https://github.com/sudeshsh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77688",
-    "pr_number": 77688,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "77807": {
-    "commit": "3ebe6a6a5fca72759a0482907238221ea3be5c82",
-    "text": "Added new startupProbe, related to KEP https://github.com/kubernetes/enhancements/issues/950.",
-    "markdown": "Added new startupProbe, related to KEP https://github.com/kubernetes/enhancements/issues/950. ([#77807](https://github.com/kubernetes/kubernetes/pull/77807), [@matthyx](https://github.com/matthyx))",
-    "author": "matthyx",
-    "author_url": "https://github.com/matthyx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77807",
-    "pr_number": 77807,
-    "areas": ["dependency", "kubectl", "kubelet", "release-eng", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["apps", "cli", "node", "release", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "77824": {
-    "commit": "e2f57be0c029cf80451983b8aaf49ed476115f6d",
-    "text": "Audit events now log the existence and patch of mutating webhooks. \n- At Metadata audit level or higher, an annotation with key \"mutation.webhook.admission.k8s.io/round_{round idx}_index_{order idx}\" gets logged with JSON payload indicating a webhook gets invoked for given request and whether it mutated the object or not.\n- At Request audit level or higher, an annotation with key \"patch.webhook.admission.k8s.io/round_{round idx}_index_{order idx}\" get logged with the JSON payload logging the patch sent by a webhook for given request.",
-    "markdown": "Audit events now log the existence and patch of mutating webhooks. \n  - At Metadata audit level or higher, an annotation with key \"mutation.webhook.admission.k8s.io/round_{round idx}_index_{order idx}\" gets logged with JSON payload indicating a webhook gets invoked for given request and whether it mutated the object or not.\n  - At Request audit level or higher, an annotation with key \"patch.webhook.admission.k8s.io/round_{round idx}_index_{order idx}\" get logged with the JSON payload logging the patch sent by a webhook for given request. ([#77824](https://github.com/kubernetes/kubernetes/pull/77824), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77824",
-    "pr_number": 77824,
-    "areas": ["admission-control", "apiserver", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "auth", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "78033": {
-    "commit": "e1c2c677f14b41fff3bb1752e08ec7cdfd535aa8",
-    "text": "Kubeadm now seamlessly migrates the CoreDNS Configuration when upgrading CoreDNS.",
-    "markdown": "Kubeadm now seamlessly migrates the CoreDNS Configuration when upgrading CoreDNS. ([#78033](https://github.com/kubernetes/kubernetes/pull/78033), [@rajansandeep](https://github.com/rajansandeep))\n\n  Courtesy of SIG API Machinery, SIG CLI, and SIG Cluster Lifecycle",
-    "author": "rajansandeep",
-    "author_url": "https://github.com/rajansandeep",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78033",
-    "pr_number": 78033,
-    "areas": ["dependency", "kubeadm", "kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cli", "cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "78098": {
-    "commit": "3cf2d32f871c73c43bd3e477583373735a368976",
-    "text": "Remove kubectl log, use kubectl logs instead",
-    "markdown": "Remove kubectl log, use kubectl logs instead ([#78098](https://github.com/kubernetes/kubernetes/pull/78098), [@soltysh](https://github.com/soltysh))",
-    "author": "soltysh",
-    "author_url": "https://github.com/soltysh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78098",
-    "pr_number": 78098,
-    "areas": ["kubectl", "test"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "78502": {
-    "commit": "f44d8f55ca1308ef576a15d264b23d4a00f77e8b",
-    "text": "Service account tokens now include the JWT Key ID field in their header.",
-    "markdown": "Service account tokens now include the JWT Key ID field in their header. ([#78502](https://github.com/kubernetes/kubernetes/pull/78502), [@ahmedtd](https://github.com/ahmedtd))\n\n  Courtesy of SIG Auth",
-    "author": "ahmedtd",
-    "author_url": "https://github.com/ahmedtd",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78502",
-    "pr_number": 78502,
-    "kinds": ["feature"],
-    "sigs": ["auth"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "78829": {
-    "commit": "f9afe46d23016a010a2c34b65fa312cde90153ad",
-    "text": "Verify that CRD default values in OpenAPI specs are pruned, with the exceptions of values under `metadata`.",
-    "markdown": "Verify that CRD default values in OpenAPI specs are pruned, with the exceptions of values under `metadata`. ([#78829](https://github.com/kubernetes/kubernetes/pull/78829), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78829",
-    "pr_number": 78829,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.7"
-  },
-  "79247": {
-    "commit": "8cf05f514c139b664f13687b8ac3c24fb99311ff",
-    "text": "Introduce support for applying pod overhead to pod cgroups, if the PodOverhead feature is enabled.",
-    "markdown": "Introduce support for applying pod overhead to pod cgroups, if the PodOverhead feature is enabled. ([#79247](https://github.com/kubernetes/kubernetes/pull/79247), [@egernst](https://github.com/egernst))\n\n  Courtesy of SIG Node",
-    "author": "egernst",
-    "author_url": "https://github.com/egernst",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79247",
-    "pr_number": 79247,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "79386": {
-    "commit": "550fb1bfc3c773febbb0811262fd7a8f7847bb20",
-    "text": "+ to run:\nMaster: convert service CIDR to list  `--service-cluster-ip-range=\u003cCIDR\u003e,\u003cCIDR\u003e` and make sure `IPv6DualStack` feature flag is turned on. The flag is validated and used as the following:\n\n1. `--service-cluster-ip-range[0]` is consider primary service range, and will be used for any service with `Service.Spec.IPFamily = nil` or any service in the at the time of turning on the feature flag.\n2. A cluster can be dualstack (i.e. Pods and nodes carry dualstack IPs) but does not need to support ingress on dualstack. In this case the cluster can perform egress using `PodIPs` (according to family and binding selection in user code) but will ingress will only be performed against the pod primary IP. This can be configured by supplying single entry to `--service-cluster-ip-range` flag.\n3. Maximum of two entries is allowed in `--service-cluster-ip-range` and they are validated to be dual stacked `i.e. --service-cluster-ip-range=\u003cv4\u003e,\u003cv6\u003e or --service-cluster-ip-range=\u003cv6\u003e,\u003cv4\u003e`  \n4. Max 20 bit for range (min network bits `\u003cv6\u003e/108` or \u003cv4\u003e/12)\n\nkube-controller-manager: convert service CIDR to list `--service-cluster-ip-range=\u003cCIDR\u003e,\u003cCIDR\u003e` and make sure `IPv6DualStack` feature flag is turned on. The flag is validated as above.\n\n+ to use:\nA new service spec field `Service.Spec.IPFamily` has been added. The default of this field is family of (first service cidr in --service-cluster-ip-range flag). The value is defaulted as described above  once the feature gate is turned on. Here are the possible values for this field:\n2. IPv4: api-server will assign an IP from a `service-cluster-ip-range` that is `ipv4` (either the primary or the secondary, according to how they were configured).\n2. IPv6: api-server will assign an IP from a `service-cluster-ip-range` that is `ipv6` (either the primary or the secondary, according to how they were configured).\n\nNotes (v1.16):\n1. IPVS is the only proxy supported (as of v1.16 ) by Dualstack.\n2. Dualstack is mutually exclusive with `EndpointSlice` feature. They can not be turned on together.  `metaproxy` is yet to implement EndpointSlice handling.",
-    "markdown": "+ to run:\n  Master: convert service CIDR to list  `--service-cluster-ip-range=\u003cCIDR\u003e,\u003cCIDR\u003e` and make sure `IPv6DualStack` feature flag is turned on. The flag is validated and used as the following:\n  \n  1. `--service-cluster-ip-range[0]` is consider primary service range, and will be used for any service with `Service.Spec.IPFamily = nil` or any service in the at the time of turning on the feature flag.\n  2. A cluster can be dualstack (i.e. Pods and nodes carry dualstack IPs) but does not need to support ingress on dualstack. In this case the cluster can perform egress using `PodIPs` (according to family and binding selection in user code) but will ingress will only be performed against the pod primary IP. This can be configured by supplying single entry to `--service-cluster-ip-range` flag.\n  3. Maximum of two entries is allowed in `--service-cluster-ip-range` and they are validated to be dual stacked `i.e. --service-cluster-ip-range=\u003cv4\u003e,\u003cv6\u003e or --service-cluster-ip-range=\u003cv6\u003e,\u003cv4\u003e`  \n  4. Max 20 bit for range (min network bits `\u003cv6\u003e/108` or \u003cv4\u003e/12)\n  \n  kube-controller-manager: convert service CIDR to list `--service-cluster-ip-range=\u003cCIDR\u003e,\u003cCIDR\u003e` and make sure `IPv6DualStack` feature flag is turned on. The flag is validated as above.\n  \n  + to use:\n  A new service spec field `Service.Spec.IPFamily` has been added. The default of this field is family of (first service cidr in --service-cluster-ip-range flag). The value is defaulted as described above  once the feature gate is turned on. Here are the possible values for this field:\n  2. IPv4: api-server will assign an IP from a `service-cluster-ip-range` that is `ipv4` (either the primary or the secondary, according to how they were configured).\n  2. IPv6: api-server will assign an IP from a `service-cluster-ip-range` that is `ipv6` (either the primary or the secondary, according to how they were configured).\n  \n  Notes (v1.16):\n  1. IPVS is the only proxy supported (as of v1.16 ) by Dualstack.\n  2. Dualstack is mutually exclusive with `EndpointSlice` feature. They can not be turned on together.  `metaproxy` is yet to implement EndpointSlice handling. ([#79386](https://github.com/kubernetes/kubernetes/pull/79386), [@khenidak](https://github.com/khenidak))",
-    "author": "khenidak",
-    "author_url": "https://github.com/khenidak",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79386",
-    "pr_number": 79386,
-    "areas": [
-      "apiserver",
-      "cloudprovider",
-      "code-generation",
-      "dependency",
-      "kubectl",
-      "kubelet",
-      "release-eng",
-      "test"
-    ],
-    "kinds": ["api-change"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "network",
-      "node",
-      "release",
-      "scheduling",
-      "testing"
-    ],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "79539": {
-    "commit": "6c45b3c23017bb232dc1db90e8fac914c9f2a705",
-    "text": "Added E2E tests validating WindowsOptions.RunAsUserName.",
-    "markdown": "Added E2E tests validating WindowsOptions.RunAsUserName. ([#79539](https://github.com/kubernetes/kubernetes/pull/79539), [@claudiubelu](https://github.com/claudiubelu))\n\n  Courtesy of SIG Apps, SIG Node, SIG Testing, and SIG Windows",
-    "author": "claudiubelu",
-    "author_url": "https://github.com/claudiubelu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79539",
-    "pr_number": 79539,
-    "areas": ["kubelet", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["apps", "node", "testing", "windows"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "79604": {
-    "commit": "ef397e5b368d530e712165a4bb78ec482d739c2b",
-    "text": "The `CustomResourceDefinition` API type is promoted to `apiextensions.k8s.io/v1` with the following changes:\n- Use of the new `default` feature in validation schemas is limited to v1\n- `spec.scope` is no longer defaulted to `Namespaced` and must be explicitly specified\n- `spec.version` is removed; use `spec.versions` instead\n- `spec.validation` is removed; use `spec.versions[*].schema` instead\n- `spec.subresources` is removed; use `spec.versions[*].subresources` instead\n- `spec.additionalPrinterColumns` is removed; use `spec.versions[*].additionalPrinterColumns` instead\n- `spec.conversion.webhookClientConfig` is moved to `spec.conversion.webhook.clientConfig`\n- `spec.conversion.conversionReviewVersions` is moved to `spec.conversion.webhook.conversionReviewVersions`\n- `spec.versions[*].schema.openAPIV3Schema` is now required when creating v1 CustomResourceDefinitions\n- `spec.preserveUnknownFields: true` is disallowed when creating v1 CustomResourceDefinitions; it must be specified within schema definitions as `x-kubernetes-preserve-unknown-fields: true`\n- In `additionalPrinterColumns` items, the `JSONPath` field was renamed to `jsonPath` (fixes https://github.com/kubernetes/kubernetes/issues/66531)\nThe `apiextensions.k8s.io/v1beta1` version of `CustomResourceDefinition` is deprecated and will no longer be served in v1.19.",
-    "markdown": "The `CustomResourceDefinition` API type is promoted to `apiextensions.k8s.io/v1` with the following changes:\n  - Use of the new `default` feature in validation schemas is limited to v1\n  - `spec.scope` is no longer defaulted to `Namespaced` and must be explicitly specified\n  - `spec.version` is removed; use `spec.versions` instead\n  - `spec.validation` is removed; use `spec.versions[*].schema` instead\n  - `spec.subresources` is removed; use `spec.versions[*].subresources` instead\n  - `spec.additionalPrinterColumns` is removed; use `spec.versions[*].additionalPrinterColumns` instead\n  - `spec.conversion.webhookClientConfig` is moved to `spec.conversion.webhook.clientConfig`\n  - `spec.conversion.conversionReviewVersions` is moved to `spec.conversion.webhook.conversionReviewVersions`\n  - `spec.versions[*].schema.openAPIV3Schema` is now required when creating v1 CustomResourceDefinitions\n  - `spec.preserveUnknownFields: true` is disallowed when creating v1 CustomResourceDefinitions; it must be specified within schema definitions as `x-kubernetes-preserve-unknown-fields: true`\n  - In `additionalPrinterColumns` items, the `JSONPath` field was renamed to `jsonPath` (fixes https://github.com/kubernetes/kubernetes/issues/66531)\n  The `apiextensions.k8s.io/v1beta1` version of `CustomResourceDefinition` is deprecated and will no longer be served in v1.19. ([#79604](https://github.com/kubernetes/kubernetes/pull/79604), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery, and SIG Testing",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79604",
-    "pr_number": 79604,
-    "areas": ["apiserver", "custom-resources", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "testing"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "80238": {
-    "commit": "92a320aeb632d6525acafd1869005a4d05560de6",
-    "text": "Introduce `node.kubernetes.io/exclude-balancer` and `node.kubernetes.io/exclude-disruption` labels in alpha to prevent cluster deployers from being dependent on the optional `node-role` labels which not all clusters may provide.",
-    "markdown": "Introduce `node.kubernetes.io/exclude-balancer` and `node.kubernetes.io/exclude-disruption` labels in alpha to prevent cluster deployers from being dependent on the optional `node-role` labels which not all clusters may provide. ([#80238](https://github.com/kubernetes/kubernetes/pull/80238), [@smarterclayton](https://github.com/smarterclayton))\n\n  Courtesy of SIG Apps, SIG Network, and SIG Testing",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80238",
-    "pr_number": 80238,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["feature"],
-    "sigs": ["apps", "network", "testing"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "80253": {
-    "commit": "18d7f8839d9330ab64a4e689d64ad7a79fe4e8a9",
-    "text": "`framework.ExpectNoError` no longer logs the error and instead relies on using the new `log.Fail` as Gomega fail handler.",
-    "markdown": "`framework.ExpectNoError` no longer logs the error and instead relies on using the new `log.Fail` as Gomega fail handler. ([#80253](https://github.com/kubernetes/kubernetes/pull/80253), [@pohly](https://github.com/pohly))\n\n  Courtesy of SIG Testing",
-    "author": "pohly",
-    "author_url": "https://github.com/pohly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80253",
-    "pr_number": 80253,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["feature"],
-    "sigs": ["testing"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "80337": {
-    "commit": "bd6f58cfcbab921d0f9431ae6cc5ad2cc0eb6e2f",
-    "text": "Remove deprecated \"include-uninitialized\" flag. action required",
-    "markdown": "Remove deprecated \"include-uninitialized\" flag. action required ([#80337](https://github.com/kubernetes/kubernetes/pull/80337), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG CLI",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80337",
-    "pr_number": 80337,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "action_required": true,
-    "release_version": "1.14.7"
-  },
-  "80368": {
-    "commit": "37651f1cef5ec5c7286ba409ee0fe298c4605d66",
-    "text": "Fixes a problem with the iptables proxy mode that could result in long delays\nupdating Service/Endpoints IPs in very large clusters on RHEL/CentOS 7.",
-    "markdown": "Fixes a problem with the iptables proxy mode that could result in long delays\n  updating Service/Endpoints IPs in very large clusters on RHEL/CentOS 7. ([#80368](https://github.com/kubernetes/kubernetes/pull/80368), [@danwinship](https://github.com/danwinship))",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80368",
-    "pr_number": 80368,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["network", "node", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "80477": {
-    "commit": "ff1e1127e27d4a144de98082c66800f2300592b0",
-    "text": "Added a metric 'apiserver_watch_events_sizes' that can be used to estimate sizes of watch events in the system.",
-    "markdown": "Added a metric 'apiserver_watch_events_sizes' that can be used to estimate sizes of watch events in the system. ([#80477](https://github.com/kubernetes/kubernetes/pull/80477), [@mborsz](https://github.com/mborsz))\n\n  Courtesy of SIG API Machinery",
-    "author": "mborsz",
-    "author_url": "https://github.com/mborsz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80477",
-    "pr_number": 80477,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "80485": {
-    "commit": "3e7e12dac83c90cc9fee1c50cb9d4460a9e14d59",
-    "text": "Azure supports IPv6 only on ELB not ILB. The cloud provider will return an error if the service is internal and is IPv6.\n\nNotes on LB name:\nto ensure backword and forward compat:\n- SingleStack -v4 (pre v1.16) =\u003e BackendPool name == clusterName\n- SingleStack -v6 =\u003e BackendPool name == clusterName (all cluster bootstrap uses this name)\nDualStack:\n=\u003e IPv4 BackendPool name == clusterName\n=\u003e IPv6 BackendPool name == \u003cclusterName\u003e-IPv6\nThis result into:\n- clusters moving from IPv4 to duakstack will require no changes\n- clusters moving from IPv6 (while not seen in the wild, we can not rule out thier existance) to dualstack will require deleting backend pools (the reconciler will take care of creating correct backendpools)",
-    "markdown": "Azure supports IPv6 only on ELB not ILB. The cloud provider will return an error if the service is internal and is IPv6.\n  \n  Notes on LB name:\n  to ensure backword and forward compat:\n  - SingleStack -v4 (pre v1.16) =\u003e BackendPool name == clusterName\n  - SingleStack -v6 =\u003e BackendPool name == clusterName (all cluster bootstrap uses this name)\n  DualStack:\n  =\u003e IPv4 BackendPool name == clusterName\n  =\u003e IPv6 BackendPool name == \u003cclusterName\u003e-IPv6\n  This result into:\n  - clusters moving from IPv4 to duakstack will require no changes\n  - clusters moving from IPv6 (while not seen in the wild, we can not rule out thier existance) to dualstack will require deleting backend pools (the reconciler will take care of creating correct backendpools) ([#80485](https://github.com/kubernetes/kubernetes/pull/80485), [@khenidak](https://github.com/khenidak))",
-    "author": "khenidak",
-    "author_url": "https://github.com/khenidak",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80485",
-    "pr_number": 80485,
-    "areas": [
-      "apiserver",
-      "cloudprovider",
-      "kubectl",
-      "kubelet",
-      "provider/azure",
-      "release-eng",
-      "test"
-    ],
-    "kinds": ["api-change"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "cli",
-      "cloud-provider",
-      "network",
-      "node",
-      "release",
-      "testing"
-    ],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "80568": {
-    "commit": "087aafc18d4631728187f014a487b30d2a0986d3",
-    "text": "a CSI driver that supports ephemeral inline volumes must explicitly declare that by providing a CSIDriver object where the new \"mode\" field is set to \"ephemeral\" or \"persistent+ephemeral\"",
-    "markdown": "a CSI driver that supports ephemeral inline volumes must explicitly declare that by providing a CSIDriver object where the new \"mode\" field is set to \"ephemeral\" or \"persistent+ephemeral\" ([#80568](https://github.com/kubernetes/kubernetes/pull/80568), [@pohly](https://github.com/pohly))",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/20190122-csi-inline-volumes.md#driver-mode",
-        "type": "KEP"
-      }
-    ],
-    "author": "pohly",
-    "author_url": "https://github.com/pohly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80568",
-    "pr_number": 80568,
-    "areas": ["test"],
-    "kinds": ["api-change"],
-    "sigs": ["storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "80696": {
-    "commit": "9aa76b2c8aef62e1ad352451444060116c052989",
-    "text": "Add a helper function to decode scheduler plugin args.",
-    "markdown": "Add a helper function to decode scheduler plugin args. ([#80696](https://github.com/kubernetes/kubernetes/pull/80696), [@hex108](https://github.com/hex108))",
-    "author": "hex108",
-    "author_url": "https://github.com/hex108",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80696",
-    "pr_number": 80696,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.7"
-  },
-  "80726": {
-    "commit": "b9c6f559bd0137c87714feea950d69f1090d14c9",
-    "text": "Node-Problem-Detector v0.7.1 is used on GCI.",
-    "markdown": "Node-Problem-Detector v0.7.1 is used on GCI. ([#80726](https://github.com/kubernetes/kubernetes/pull/80726), [@wangzhen127](https://github.com/wangzhen127))\n\n  Courtesy of SIG Cluster Lifecycle, SIG Node, and SIG Testing",
-    "author": "wangzhen127",
-    "author_url": "https://github.com/wangzhen127",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80726",
-    "pr_number": 80726,
-    "areas": ["dependency", "test"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "node", "testing"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "80773": {
-    "commit": "3dbedefb80a54e39e6496853a8ac6f98d7a08757",
-    "text": "Fixed a bug with the openAPI definition for io.k8s.apimachinery.pkg.runtime.RawExtension, which previously required a field \"raw\" to be specified",
-    "markdown": "Fixed a bug with the openAPI definition for io.k8s.apimachinery.pkg.runtime.RawExtension, which previously required a field \"raw\" to be specified ([#80773](https://github.com/kubernetes/kubernetes/pull/80773), [@jennybuckley](https://github.com/jennybuckley))",
-    "author": "jennybuckley",
-    "author_url": "https://github.com/jennybuckley",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80773",
-    "pr_number": 80773,
-    "areas": ["code-generation"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.7"
-  },
-  "80911": {
-    "commit": "77277d3abf9cd7289480c6c353da11ba7db9cc79",
-    "text": "Adds support for vSphere volumes on Windows",
-    "markdown": "Adds support for vSphere volumes on Windows ([#80911](https://github.com/kubernetes/kubernetes/pull/80911), [@gab-satchi](https://github.com/gab-satchi))\n\n  Courtesy of SIG Storage, and SIG Windows",
-    "author": "gab-satchi",
-    "author_url": "https://github.com/gab-satchi",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80911",
-    "pr_number": 80911,
-    "areas": ["provider/vmware"],
-    "kinds": ["feature"],
-    "sigs": ["storage", "windows"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "80912": {
-    "commit": "6789f38199d20cf8fadf38d79b7484d8af0a6ebd",
-    "text": "updates fluentd-elasticsearch docker image to fluentd 1.6.3",
-    "markdown": "updates fluentd-elasticsearch docker image to fluentd 1.6.3 ([#80912](https://github.com/kubernetes/kubernetes/pull/80912), [@monotek](https://github.com/monotek))",
-    "author": "monotek",
-    "author_url": "https://github.com/monotek",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80912",
-    "pr_number": 80912,
-    "kinds": ["flake"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "80930": {
-    "commit": "1a53325550f6d5d3c48b9eecdd123fd84deee879",
-    "text": "NormalizeScore plugin set is removed from scheduler framework config API. Use ScorePlugin only.",
-    "markdown": "NormalizeScore plugin set is removed from scheduler framework config API. Use ScorePlugin only. ([#80930](https://github.com/kubernetes/kubernetes/pull/80930), [@liu-cong](https://github.com/liu-cong))",
-    "author": "liu-cong",
-    "author_url": "https://github.com/liu-cong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80930",
-    "pr_number": 80930,
-    "kinds": ["api-change"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.7"
-  },
-  "80937": {
-    "commit": "59f5b3ddc4ddcc7d400d3fd5624712e79929bffc",
-    "text": "kubeadm: print the stack trace of an error for klog level --v\u003e=5",
-    "markdown": "kubeadm: print the stack trace of an error for klog level --v\u003e=5 ([#80937](https://github.com/kubernetes/kubernetes/pull/80937), [@neolit123](https://github.com/neolit123))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80937",
-    "pr_number": 80937,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81043": {
-    "commit": "0f46a8a4c87a092e536b5af538dde60b0cf77736",
-    "text": "/test/e2e/framework: Adds a flag \"non-blocking-taints\" which allows tests to run in environments with tainted nodes. String value should be a comma-separated list.",
-    "markdown": "/test/e2e/framework: Adds a flag \"non-blocking-taints\" which allows tests to run in environments with tainted nodes. String value should be a comma-separated list. ([#81043](https://github.com/kubernetes/kubernetes/pull/81043), [@johnSchnake](https://github.com/johnSchnake))\n\n  Courtesy of SIG Testing",
-    "author": "johnSchnake",
-    "author_url": "https://github.com/johnSchnake",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81043",
-    "pr_number": 81043,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["feature"],
-    "sigs": ["testing"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81048": {
-    "commit": "7da9bdc4f64a5014a4b45cc34c8a86976db1ce11",
-    "text": "Adds Endpoint Slice Controller for managing new EndpointSlice resource, disabled by default.",
-    "markdown": "Adds Endpoint Slice Controller for managing new EndpointSlice resource, disabled by default. ([#81048](https://github.com/kubernetes/kubernetes/pull/81048), [@robscott](https://github.com/robscott))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG Network, and SIG Testing",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81048",
-    "pr_number": 81048,
-    "areas": ["apiserver", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "auth", "network", "testing"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81163": {
-    "commit": "e319abf2745b5c70f55a2b8b03bfac56a5ee088f",
-    "text": "Volumes specified in a pod but not used in it are no longer unnecessarily formatted, mounted and reported in `node.status.volumesInUse`.",
-    "markdown": "Volumes specified in a pod but not used in it are no longer unnecessarily formatted, mounted and reported in `node.status.volumesInUse`. ([#81163](https://github.com/kubernetes/kubernetes/pull/81163), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81163",
-    "pr_number": 81163,
-    "areas": ["e2e-test-framework", "kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["node", "storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "81266": {
-    "commit": "1719ce7883db57d24d70605606f189469fd50b60",
-    "text": "fix: detach azure disk issue using dangling error",
-    "markdown": "fix: detach azure disk issue using dangling error ([#81266](https://github.com/kubernetes/kubernetes/pull/81266), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81266",
-    "pr_number": 81266,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "81300": {
-    "commit": "9614a85a2b6b3122b9ca68c4992801648248dcd3",
-    "text": "Install and start logging agent based on kube env ENABLE_NODE_LOGGING.",
-    "markdown": "Install and start logging agent based on kube env ENABLE_NODE_LOGGING. ([#81300](https://github.com/kubernetes/kubernetes/pull/81300), [@liyanhui1228](https://github.com/liyanhui1228))",
-    "author": "liyanhui1228",
-    "author_url": "https://github.com/liyanhui1228",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81300",
-    "pr_number": 81300,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "81342": {
-    "commit": "f81ab5a9fca3212386fd1eb0fb40950f40f55a33",
-    "text": "kubectl could scale custom resource again",
-    "markdown": "kubectl could scale custom resource again ([#81342](https://github.com/kubernetes/kubernetes/pull/81342), [@knight42](https://github.com/knight42))\n\n  Courtesy of SIG CLI, and SIG Testing",
-    "author": "knight42",
-    "author_url": "https://github.com/knight42",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81342",
-    "pr_number": 81342,
-    "areas": ["e2e-test-framework", "kubectl", "test"],
-    "kinds": ["feature"],
-    "sigs": ["cli", "testing"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81397": {
-    "commit": "08b67378d3557734caab683e856c8e09ce1d7876",
-    "text": "Support Kubelet plugin watcher on Windows nodes.",
-    "markdown": "Support Kubelet plugin watcher on Windows nodes. ([#81397](https://github.com/kubernetes/kubernetes/pull/81397), [@ddebroy](https://github.com/ddebroy))\n\n  Courtesy of SIG Node, and SIG Windows",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-windows/20190714-windows-csi-support.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "ddebroy",
-    "author_url": "https://github.com/ddebroy",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81397",
-    "pr_number": 81397,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node", "windows"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81399": {
-    "commit": "34605737b9a96ee4b94ebf8f970501e238086370",
-    "text": "The `rejected` label in `apiserver_admission_webhook_admission_duration_seconds` metrices now properly indicates if a request was rejected. Add a new counter metrics `apiserver_admission_webhook_rejection_count` with details about the causing for a webhook rejection.",
-    "markdown": "The `rejected` label in `apiserver_admission_webhook_admission_duration_seconds` metrices now properly indicates if a request was rejected. Add a new counter metrics `apiserver_admission_webhook_rejection_count` with details about the causing for a webhook rejection. ([#81399](https://github.com/kubernetes/kubernetes/pull/81399), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81399",
-    "pr_number": 81399,
-    "areas": ["admission-control", "apiserver"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "81411": {
-    "commit": "11eaf430e1ed0b4bc18dbc18321c99a94ee96911",
-    "text": "Fix VMSS LoadBalancer backend pools so that the network won't be broken when instances are upgraded to latest model",
-    "markdown": "Fix VMSS LoadBalancer backend pools so that the network won't be broken when instances are upgraded to latest model ([#81411](https://github.com/kubernetes/kubernetes/pull/81411), [@nilo19](https://github.com/nilo19))",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81411",
-    "pr_number": 81411,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.7"
-  },
-  "81413": {
-    "commit": "443002fcb7bdd5261e383b6da358a16e43183415",
-    "text": "The nbf (not before) claim, if present in ID token, is now enforced.",
-    "markdown": "The nbf (not before) claim, if present in ID token, is now enforced. ([#81413](https://github.com/kubernetes/kubernetes/pull/81413), [@anderseknert](https://github.com/anderseknert))\n\n  Courtesy of SIG API Machinery, SIG Auth, and SIG Cloud Provider",
-    "author": "anderseknert",
-    "author_url": "https://github.com/anderseknert",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81413",
-    "pr_number": 81413,
-    "areas": ["apiserver", "cloudprovider", "dependency"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "auth", "cloud-provider"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81429": {
-    "commit": "f105fef3d5ca0c78b056bf826ee546bf1107cf8b",
-    "text": "Resolves an issue that prevented block volumes from being resized.",
-    "markdown": "Resolves an issue that prevented block volumes from being resized. ([#81429](https://github.com/kubernetes/kubernetes/pull/81429), [@huffmanca](https://github.com/huffmanca))",
-    "author": "huffmanca",
-    "author_url": "https://github.com/huffmanca",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81429",
-    "pr_number": 81429,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "81430": {
-    "commit": "4495d09282265964ae217ab9976c4c4ff5945b7b",
-    "text": "Adds EndpointSlice integration to kube-proxy, can be enabled with EndpointSlice feature gate.",
-    "markdown": "Adds EndpointSlice integration to kube-proxy, can be enabled with EndpointSlice feature gate. ([#81430](https://github.com/kubernetes/kubernetes/pull/81430), [@robscott](https://github.com/robscott))\n\n  Courtesy of SIG API Machinery, SIG Network, and SIG Testing",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81430",
-    "pr_number": 81430,
-    "areas": ["apiserver", "ipvs", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "network", "testing"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81436": {
-    "commit": "21cdcd577609c039043903edfd4995077fc6b27d",
-    "text": "Fix a bug in CRD openapi controller that user-defined CRD can overwrite OpenAPI definition/path for the CRD API.",
-    "markdown": "Fix a bug in CRD openapi controller that user-defined CRD can overwrite OpenAPI definition/path for the CRD API. ([#81436](https://github.com/kubernetes/kubernetes/pull/81436), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81436",
-    "pr_number": 81436,
-    "areas": ["custom-resources", "dependency", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "81467": {
-    "commit": "dccd28269a6596bf2f0041c14dd3233be8cfa571",
-    "text": "Move CSI volume expansion to beta.",
-    "markdown": "Move CSI volume expansion to beta. ([#81467](https://github.com/kubernetes/kubernetes/pull/81467), [@bertinatto](https://github.com/bertinatto))\n\n  Courtesy of SIG Testing",
-    "documentation": [
-      {
-        "url": "https://github.com/kubernetes/enhancements/issues/556",
-        "type": "KEP"
-      }
-    ],
-    "author": "bertinatto",
-    "author_url": "https://github.com/bertinatto",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81467",
-    "pr_number": 81467,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["testing"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81468": {
-    "commit": "e102d575f4625a1d5bd1178391d84c5559df1dbe",
-    "text": "kubectl: the --all-namespaces flag is now honored by `kubectl wait`",
-    "markdown": "kubectl: the --all-namespaces flag is now honored by `kubectl wait` ([#81468](https://github.com/kubernetes/kubernetes/pull/81468), [@ashutoshgngwr](https://github.com/ashutoshgngwr))",
-    "author": "ashutoshgngwr",
-    "author_url": "https://github.com/ashutoshgngwr",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81468",
-    "pr_number": 81468,
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.7"
-  },
-  "81476": {
-    "commit": "770303eaee8cc50374975889e1fd1d2db869a8f8",
-    "text": "Conversion webhooks can now indicate they support receiving and responding with `ConversionReview` API objects in the `apiextensions.k8s.io/v1` version by including `v1` in the `conversionReviewVersions` list in their CustomResourceDefinition. Conversion webhooks must respond with a ConversionReview object in the same apiVersion they receive. `apiextensions.k8s.io/v1` `ConversionReview` responses must specify a `response.uid` that matches the `request.uid` of the object they were sent.",
-    "markdown": "Conversion webhooks can now indicate they support receiving and responding with `ConversionReview` API objects in the `apiextensions.k8s.io/v1` version by including `v1` in the `conversionReviewVersions` list in their CustomResourceDefinition. Conversion webhooks must respond with a ConversionReview object in the same apiVersion they receive. `apiextensions.k8s.io/v1` `ConversionReview` responses must specify a `response.uid` that matches the `request.uid` of the object they were sent. ([#81476](https://github.com/kubernetes/kubernetes/pull/81476), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery, and SIG Testing",
-    "documentation": [
-      {
-        "description": "KEP",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190425-crd-conversion-webhook.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81476",
-    "pr_number": 81476,
-    "areas": ["apiserver", "custom-resources", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "testing"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81477": {
-    "commit": "bd8a8db515381b3058e211822a00040662109911",
-    "text": "Fix in kube-proxy for SCTP nodeport service which only works for node's InternalIP, but doesn't work for other IPs present in the node when ipvs is enabled.",
-    "markdown": "Fix in kube-proxy for SCTP nodeport service which only works for node's InternalIP, but doesn't work for other IPs present in the node when ipvs is enabled. ([#81477](https://github.com/kubernetes/kubernetes/pull/81477), [@paulsubrata55](https://github.com/paulsubrata55))",
-    "author": "paulsubrata55",
-    "author_url": "https://github.com/paulsubrata55",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81477",
-    "pr_number": 81477,
-    "areas": ["ipvs"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.7"
-  },
-  "81494": {
-    "commit": "17a1859370785fe94150b7489215c1a2aea24dfa",
-    "text": "kubeadm reset: unmount directories under \"/var/lib/kubelet\" for linux only",
-    "markdown": "kubeadm reset: unmount directories under \"/var/lib/kubelet\" for linux only ([#81494](https://github.com/kubernetes/kubernetes/pull/81494), [@Klaven](https://github.com/Klaven))",
-    "author": "Klaven",
-    "author_url": "https://github.com/Klaven",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81494",
-    "pr_number": 81494,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.7"
-  },
-  "81500": {
-    "commit": "667ea63ec2f96525f45848f0b6a60b512b3911a7",
-    "text": "Kubelet could be run with no Azure identity without subscriptionId configured now.\nA sample cloud provider configure is: '{\"vmType\": \"vmss\", \"useInstanceMetadata\": true}'.",
-    "markdown": "Kubelet could be run with no Azure identity without subscriptionId configured now.\n  A sample cloud provider configure is: '{\"vmType\": \"vmss\", \"useInstanceMetadata\": true}'. ([#81500](https://github.com/kubernetes/kubernetes/pull/81500), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81500",
-    "pr_number": 81500,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.7"
-  },
-  "81509": {
-    "commit": "30f2545364f3225b62651e26b872e5abdf21ab0b",
-    "text": "Added metrics 'authentication_attempts' that can be used to understand the attempts of authentication.",
-    "markdown": "Added metrics 'authentication_attempts' that can be used to understand the attempts of authentication. ([#81509](https://github.com/kubernetes/kubernetes/pull/81509), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG API Machinery, SIG Auth, and SIG Instrumentation",
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81509",
-    "pr_number": 81509,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "auth", "instrumentation"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81515": {
-    "commit": "c7e997b5271bdf7d27fdd7957361bc12133b35c5",
-    "text": "openapi now advertises correctly supported patch types for custom resources",
-    "markdown": "openapi now advertises correctly supported patch types for custom resources ([#81515](https://github.com/kubernetes/kubernetes/pull/81515), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81515",
-    "pr_number": 81515,
-    "areas": ["custom-resources"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.7"
-  },
-  "81531": {
-    "commit": "f24aa47ceab19d174117d80eec657875c06aeb0c",
-    "text": "Kube-apiserver metrics are now marked as with the ALPHA stability level.",
-    "markdown": "Kube-apiserver metrics are now marked as with the ALPHA stability level. ([#81531](https://github.com/kubernetes/kubernetes/pull/81531), [@logicalhan](https://github.com/logicalhan))\n\n  Courtesy of SIG API Machinery, SIG Auth, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Instrumentation, and SIG Release",
-    "documentation": [
-      {
-        "description": "[KEP Metrics Stability Migration]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190605-metrics-stability-migration.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81531",
-    "pr_number": 81531,
-    "areas": ["apiserver", "cloudprovider", "dependency", "release-eng"],
-    "kinds": ["feature"],
-    "sigs": [
-      "api-machinery",
-      "auth",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "release"
-    ],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81534": {
-    "commit": "ca5babc1da4f02998ceab90e3d2de57614069c45",
-    "text": "Kubelet metrics for /metrics and /metrics/probes are now marked as with the ALPHA stability level.",
-    "markdown": "Kubelet metrics for /metrics and /metrics/probes are now marked as with the ALPHA stability level. ([#81534](https://github.com/kubernetes/kubernetes/pull/81534), [@logicalhan](https://github.com/logicalhan))\n\n  Courtesy of SIG API Machinery, SIG Auth, SIG Cluster Lifecycle, SIG Instrumentation, SIG Network, SIG Node, SIG Release, and SIG Storage",
-    "documentation": [
-      {
-        "description": "[KEP Metrics Stability Migration]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190605-metrics-stability-migration.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81534",
-    "pr_number": 81534,
-    "areas": ["dependency", "kubelet", "release-eng"],
-    "kinds": ["feature"],
-    "sigs": [
-      "api-machinery",
-      "auth",
-      "cluster-lifecycle",
-      "instrumentation",
-      "network",
-      "node",
-      "release",
-      "storage"
-    ],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81549": {
-    "commit": "a7e9704c0e8ab6584e2177d59ddd2fc733af3b63",
-    "text": "Added support to specify a global-access annotation for gce ILB.",
-    "markdown": "Added support to specify a global-access annotation for gce ILB. ([#81549](https://github.com/kubernetes/kubernetes/pull/81549), [@prameshj](https://github.com/prameshj))",
-    "author": "prameshj",
-    "author_url": "https://github.com/prameshj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81549",
-    "pr_number": 81549,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "node"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "81573": {
-    "commit": "7da563f0f80d45b21e9008691c142c7f75350cc7",
-    "text": "Add `container_state` label to `running_container_count` kubelet metrics, to get count of containers based on their state(running/exited/created/unknown)",
-    "markdown": "Add `container_state` label to `running_container_count` kubelet metrics, to get count of containers based on their state(running/exited/created/unknown) ([#81573](https://github.com/kubernetes/kubernetes/pull/81573), [@irajdeep](https://github.com/irajdeep))",
-    "author": "irajdeep",
-    "author_url": "https://github.com/irajdeep",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81573",
-    "pr_number": 81573,
-    "areas": ["kubelet"],
-    "kinds": ["design"],
-    "sigs": ["instrumentation", "node"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "81576": {
-    "commit": "a927ed454cee3eea5f5acf9ff837075fb372dd0c",
-    "text": "Scheduler metrics are now marked as with the ALPHA stability level.",
-    "markdown": "Scheduler metrics are now marked as with the ALPHA stability level. ([#81576](https://github.com/kubernetes/kubernetes/pull/81576), [@logicalhan](https://github.com/logicalhan))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Instrumentation, SIG Release, and SIG Scheduling",
-    "documentation": [
-      {
-        "description": "[KEP Metrics Stability Migration]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190605-metrics-stability-migration.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81576",
-    "pr_number": 81576,
-    "areas": ["apiserver", "cloudprovider", "dependency", "release-eng"],
-    "kinds": ["feature"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "auth",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "release",
-      "scheduling"
-    ],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81623": {
-    "commit": "ca3e86e662fda8310c071e4660ef5703997f1409",
-    "text": "kubeadm: support any Linux kernel version newer than 3.10",
-    "markdown": "kubeadm: support any Linux kernel version newer than 3.10 ([#81623](https://github.com/kubernetes/kubernetes/pull/81623), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81623",
-    "pr_number": 81623,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.7"
-  },
-  "81624": {
-    "commit": "d6bc4eb853327437b25f34119a78015ca9565809",
-    "text": "Kube-controller-manager and cloud-controller-manager metrics are now marked as with the ALPHA stability level.",
-    "markdown": "Kube-controller-manager and cloud-controller-manager metrics are now marked as with the ALPHA stability level. ([#81624](https://github.com/kubernetes/kubernetes/pull/81624), [@logicalhan](https://github.com/logicalhan))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Instrumentation, SIG Release, and SIG Storage",
-    "documentation": [
-      {
-        "description": "[KEP Metrics Stability Migration]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190605-metrics-stability-migration.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81624",
-    "pr_number": 81624,
-    "areas": ["apiserver", "cloudprovider", "dependency", "release-eng"],
-    "kinds": ["feature"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "auth",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "release",
-      "storage"
-    ],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81625": {
-    "commit": "5e649417c6e8b9baf836f09a581b342dfc424430",
-    "text": "Windows nodes on GCE now run with Windows Defender enabled.",
-    "markdown": "Windows nodes on GCE now run with Windows Defender enabled. ([#81625](https://github.com/kubernetes/kubernetes/pull/81625), [@pjh](https://github.com/pjh))",
-    "author": "pjh",
-    "author_url": "https://github.com/pjh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81625",
-    "pr_number": 81625,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "81626": {
-    "commit": "7d241f0ec1b284643f2c0426f7fe62863328a2e9",
-    "text": "Kube-proxy metrics are now marked as with the ALPHA stability level.",
-    "markdown": "Kube-proxy metrics are now marked as with the ALPHA stability level. ([#81626](https://github.com/kubernetes/kubernetes/pull/81626), [@logicalhan](https://github.com/logicalhan))\n\n  Courtesy of SIG API Machinery, SIG Cluster Lifecycle, SIG Instrumentation, SIG Network, SIG Node, and SIG Release",
-    "documentation": [
-      {
-        "description": "[KEP Metrics Stability Migration]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190605-metrics-stability-migration.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81626",
-    "pr_number": 81626,
-    "areas": ["dependency", "kubelet", "release-eng"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "instrumentation", "network", "node", "release"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81634": {
-    "commit": "22a8bcf30af52d87a0289bb705ff5c41aaf6f72a",
-    "text": "Fixes a bug that when there is a \"connection refused\" error, the reflector's ListAndWatch func will return directly but what expected is that sleep 1 second and rewatch since the specified resourceVersion.",
-    "markdown": "Fixes a bug that when there is a \"connection refused\" error, the reflector's ListAndWatch func will return directly but what expected is that sleep 1 second and rewatch since the specified resourceVersion. ([#81634](https://github.com/kubernetes/kubernetes/pull/81634), [@likakuli](https://github.com/likakuli))",
-    "author": "likakuli",
-    "author_url": "https://github.com/likakuli",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81634",
-    "pr_number": 81634,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.7"
-  },
-  "81668": {
-    "commit": "1d016cc1d3ee2f95c5ea13e5666bb78d66c7c5e8",
-    "text": "Deprecate the default service IP CIDR. The previous default was `10.0.0.0/24` which will be removed in 6 months/2 releases. Cluster admins must specify their own desired value, by using `--service-cluster-ip-range` on kube-apiserver.",
-    "markdown": "Deprecate the default service IP CIDR. The previous default was `10.0.0.0/24` which will be removed in 6 months/2 releases. Cluster admins must specify their own desired value, by using `--service-cluster-ip-range` on kube-apiserver. ([#81668](https://github.com/kubernetes/kubernetes/pull/81668), [@darshanime](https://github.com/darshanime))\n\n  Courtesy of SIG API Machinery, and SIG Network",
-    "author": "darshanime",
-    "author_url": "https://github.com/darshanime",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81668",
-    "pr_number": 81668,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "network"],
-    "action_required": true,
-    "release_version": "1.14.7"
-  },
-  "81682": {
-    "commit": "36e0ea6bfaa9da19eff5e65f48b47f8f42efdcd6",
-    "text": "The RemainingItemCount feature is now beta.\n\nremainingItemCount is the number of subsequent items in the list which are not included in this list response. If the list request contained label or field selectors, then the number of remaining items is unknown and the field will be left unset and omitted during serialization. If the list is complete (either because it is not chunking or because this is the last chunk), then there are no more remaining items and this field will be left unset and omitted during serialization. Servers older than v1.15 do not set this field. The intended use of the remainingItemCount is *estimating- the size of a collection. Clients should not rely on the remainingItemCount to be set or to be exact.",
-    "markdown": "The RemainingItemCount feature is now beta.\n  \n  remainingItemCount is the number of subsequent items in the list which are not included in this list response. If the list request contained label or field selectors, then the number of remaining items is unknown and the field will be left unset and omitted during serialization. If the list is complete (either because it is not chunking or because this is the last chunk), then there are no more remaining items and this field will be left unset and omitted during serialization. Servers older than v1.15 do not set this field. The intended use of the remainingItemCount is *estimating- the size of a collection. Clients should not rely on the remainingItemCount to be set or to be exact. ([#81682](https://github.com/kubernetes/kubernetes/pull/81682), [@caesarxuchao](https://github.com/caesarxuchao))",
-    "author": "caesarxuchao",
-    "author_url": "https://github.com/caesarxuchao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81682",
-    "pr_number": 81682,
-    "areas": ["apiserver", "code-generation", "kubectl"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "cli"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "81691": {
-    "commit": "ab13cf1055098ce0ede00e9228e34ced098a9ab4",
-    "text": "Finalizer Protection for Service LoadBalancers is now in Beta (enabled by default). This feature ensures the Service resource is not fully deleted until the correlating load balancer resources are deleted.",
-    "markdown": "Finalizer Protection for Service LoadBalancers is now in Beta (enabled by default). This feature ensures the Service resource is not fully deleted until the correlating load balancer resources are deleted. ([#81691](https://github.com/kubernetes/kubernetes/pull/81691), [@MrHohn](https://github.com/MrHohn))\n\n  Courtesy of SIG Apps, SIG Network, and SIG Testing",
-    "author": "MrHohn",
-    "author_url": "https://github.com/MrHohn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81691",
-    "pr_number": 81691,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["feature"],
-    "sigs": ["apps", "network", "testing"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81720": {
-    "commit": "34450e9a2a5abd2a58980a813c4445c9175c0eab",
-    "text": "fix azure disk naming matching issue due to case sensitive comparison",
-    "markdown": "fix azure disk naming matching issue due to case sensitive comparison ([#81720](https://github.com/kubernetes/kubernetes/pull/81720), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81720",
-    "pr_number": 81720,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.7"
-  },
-  "81732": {
-    "commit": "0eb1bfc9b57e4915f205048757c98d5319e615e1",
-    "text": "Fix an issue with toleration merging \u0026 whitelist checking in the PodTolerationRestriction admission controller.",
-    "markdown": "Fix an issue with toleration merging \u0026 whitelist checking in the PodTolerationRestriction admission controller. ([#81732](https://github.com/kubernetes/kubernetes/pull/81732), [@tallclair](https://github.com/tallclair))",
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81732",
-    "pr_number": 81732,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.7"
-  },
-  "81747": {
-    "commit": "0e1bad37641c8ad790f470ed56c814cdb4e29dd2",
-    "text": "Fix `kubectl logs -f` for windows server containers.",
-    "markdown": "Fix `kubectl logs -f` for windows server containers. ([#81747](https://github.com/kubernetes/kubernetes/pull/81747), [@Random-Liu](https://github.com/Random-Liu))",
-    "author": "Random-Liu",
-    "author_url": "https://github.com/Random-Liu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81747",
-    "pr_number": 81747,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "81786": {
-    "commit": "bfd8610ddafb2fcb2cb330187c8248777ea0b022",
-    "text": "Log when kube-apiserver regenerates the OpenAPI spec and why. OpenAPI spec generation is a very CPU-heavy process that is sensitive to continuous updates of CRDs and APIServices.\n\nAdded metrics aggregator_openapi_v2_regeneration_count, aggregator_openapi_v2_regeneration_gauge and apiextension_openapi_v2_regeneration_count metrics counting the triggering APIService and CRDs and the reason (add, update, delete).",
-    "markdown": "Log when kube-apiserver regenerates the OpenAPI spec and why. OpenAPI spec generation is a very CPU-heavy process that is sensitive to continuous updates of CRDs and APIServices.\n  \n  Added metrics aggregator_openapi_v2_regeneration_count, aggregator_openapi_v2_regeneration_gauge and apiextension_openapi_v2_regeneration_count metrics counting the triggering APIService and CRDs and the reason (add, update, delete). ([#81786](https://github.com/kubernetes/kubernetes/pull/81786), [@sttts](https://github.com/sttts))\n\n  Courtesy of SIG API Machinery",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81786",
-    "pr_number": 81786,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81791": {
-    "commit": "90cf1891526ca8a131f59ac61e12f2855451f759",
-    "text": "kubeadm no longer performs IPVS checks as part of its preflight checks",
-    "markdown": "kubeadm no longer performs IPVS checks as part of its preflight checks ([#81791](https://github.com/kubernetes/kubernetes/pull/81791), [@yastij](https://github.com/yastij))",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81791",
-    "pr_number": 81791,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.7"
-  },
-  "81792": {
-    "commit": "b36265524286bd454aab36dbce944e4599dd2bf7",
-    "text": "Promotes VolumePVCDataSource (Cloning) feature to beta for 1.16 release",
-    "markdown": "Promotes VolumePVCDataSource (Cloning) feature to beta for 1.16 release ([#81792](https://github.com/kubernetes/kubernetes/pull/81792), [@j-griffith](https://github.com/j-griffith))\n\n  Courtesy of SIG Storage, and SIG Testing",
-    "author": "j-griffith",
-    "author_url": "https://github.com/j-griffith",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81792",
-    "pr_number": 81792,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["storage", "testing"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81795": {
-    "commit": "0be2f49268cab3154597794c29238f2c362e0d11",
-    "text": "Adds Endpoint Slice support for kubectl when discovery API group is enabled.",
-    "markdown": "Adds Endpoint Slice support for kubectl when discovery API group is enabled. ([#81795](https://github.com/kubernetes/kubernetes/pull/81795), [@robscott](https://github.com/robscott))\n\n  Courtesy of SIG API Machinery, SIG CLI, and SIG Testing",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81795",
-    "pr_number": 81795,
-    "areas": ["apiserver", "kubectl", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "cli", "testing"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81800": {
-    "commit": "3ede02028fe99222c6c905aeabc9d2b385c04120",
-    "text": "Use PostFilter instead of Postfilter in the scheduling framework\nUse PreFilter instead of Prefilter in the scheduling framework\nUse PreBind instead of Prebind in the scheduling framework",
-    "markdown": "Use PostFilter instead of Postfilter in the scheduling framework\n  Use PreFilter instead of Prefilter in the scheduling framework\n  Use PreBind instead of Prebind in the scheduling framework ([#81800](https://github.com/kubernetes/kubernetes/pull/81800), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81800",
-    "pr_number": 81800,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "81872": {
-    "commit": "45b0b784461f8146a539aacaa6a4006fb4a83013",
-    "text": "The CustomResourceDefaulting feature is promoted to beta and enabled by default. Defaults may be specified in structural schemas via the `apiextensions.k8s.io/v1` API. See https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/\u0026#35;specifying-a-structural-schema for details.",
-    "markdown": "The CustomResourceDefaulting feature is promoted to beta and enabled by default. Defaults may be specified in structural schemas via the `apiextensions.k8s.io/v1` API. See https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/\u0026#35;specifying-a-structural-schema for details. ([#81872](https://github.com/kubernetes/kubernetes/pull/81872), [@sttts](https://github.com/sttts))\n\n  Courtesy of SIG API Machinery",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81872",
-    "pr_number": 81872,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81876": {
-    "commit": "c58767b1e2981bd7826b4336de0c2756fa975339",
-    "text": "Fix filter plugins are not been called during preemption",
-    "markdown": "Fix filter plugins are not been called during preemption ([#81876](https://github.com/kubernetes/kubernetes/pull/81876), [@wgliang](https://github.com/wgliang))",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81876",
-    "pr_number": 81876,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.7"
-  },
-  "81903": {
-    "commit": "b3b4305be5588763d39fa96d2f81a19463fb50fa",
-    "text": "kubeadm: prevent overriding of certain kubelet security configuration parameters if the user wished to modify them",
-    "markdown": "kubeadm: prevent overriding of certain kubelet security configuration parameters if the user wished to modify them ([#81903](https://github.com/kubernetes/kubernetes/pull/81903), [@jfbai](https://github.com/jfbai))",
-    "author": "jfbai",
-    "author_url": "https://github.com/jfbai",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81903",
-    "pr_number": 81903,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.7"
-  },
-  "81929": {
-    "commit": "3d17fd5c4f6b1ed22499c4db192e1ed6a867d579",
-    "text": "Add PodOverhead awareness to kubectl",
-    "markdown": "Add PodOverhead awareness to kubectl ([#81929](https://github.com/kubernetes/kubernetes/pull/81929), [@egernst](https://github.com/egernst))\n\n  Courtesy of SIG CLI, and SIG Scheduling",
-    "author": "egernst",
-    "author_url": "https://github.com/egernst",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81929",
-    "pr_number": 81929,
-    "areas": ["kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["cli", "scheduling"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81946": {
-    "commit": "8af33f9733d8905c76fd3db85b541943b89f67d7",
-    "text": "cache-control headers are now set appropriately.  Only openapi is cacheable if etags match.",
-    "markdown": "cache-control headers are now set appropriately.  Only openapi is cacheable if etags match. ([#81946](https://github.com/kubernetes/kubernetes/pull/81946), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81946",
-    "pr_number": 81946,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "81956": {
-    "commit": "991d71e9fe7da2035e06101f1b47cb8544592335",
-    "text": "Server-side apply is now Beta.",
-    "markdown": "Server-side apply is now Beta. ([#81956](https://github.com/kubernetes/kubernetes/pull/81956), [@apelisse](https://github.com/apelisse))\n\n  Courtesy of SIG API Machinery, SIG CLI, and SIG Testing",
-    "author": "apelisse",
-    "author_url": "https://github.com/apelisse",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81956",
-    "pr_number": 81956,
-    "areas": ["apiserver", "kubectl", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cli", "testing"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81965": {
-    "commit": "f3828b776bf270290f318c0860ab5f5c8b32b2fa",
-    "text": "The `CustomResourceValidation`, `CustomResourceSubresources`, `CustomResourceWebhookConversion` and `CustomResourcePublishOpenAPI` features are now GA, and the associated feature gates deprecated and will be removed in v1.18.",
-    "markdown": "The `CustomResourceValidation`, `CustomResourceSubresources`, `CustomResourceWebhookConversion` and `CustomResourcePublishOpenAPI` features are now GA, and the associated feature gates deprecated and will be removed in v1.18. ([#81965](https://github.com/kubernetes/kubernetes/pull/81965), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81965",
-    "pr_number": 81965,
-    "areas": ["custom-resources"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.7"
-  },
-  "81969": {
-    "commit": "7acb066dbcd7d05106609711fc4ec110fec1d1a4",
-    "text": "Adds \\livez for liveness health checking for kube-apiserver. Using the parameter `--maximum-startup-sequence-duration` will allow the liveness endpoint to defer boot-sequence failures for the specified duration period.",
-    "markdown": "Adds \\livez for liveness health checking for kube-apiserver. Using the parameter `--maximum-startup-sequence-duration` will allow the liveness endpoint to defer boot-sequence failures for the specified duration period. ([#81969](https://github.com/kubernetes/kubernetes/pull/81969), [@logicalhan](https://github.com/logicalhan))\n\n  Courtesy of SIG API Machinery, SIG Auth, and SIG Testing",
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81969",
-    "pr_number": 81969,
-    "areas": ["apiserver", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "auth", "testing"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "81972": {
-    "commit": "fe770ad612cc86eef6341945ef19bde12be50a2d",
-    "text": "- Fix disk stats in LXD using ZFS storage pool\n- Fix CRI-O missing network metris bug\n- Add `container_sockets`, `container_threads`, and `container_threads_max` metrics",
-    "markdown": "- Fix disk stats in LXD using ZFS storage pool\n  - Fix CRI-O missing network metris bug\n  - Add `container_sockets`, `container_threads`, and `container_threads_max` metrics ([#81972](https://github.com/kubernetes/kubernetes/pull/81972), [@dashpole](https://github.com/dashpole))",
-    "author": "dashpole",
-    "author_url": "https://github.com/dashpole",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81972",
-    "pr_number": 81972,
-    "areas": ["dependency"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.7"
-  },
-  "82004": {
-    "commit": "cd9519fe9588b66c68f6b125cbd9d776b845951c",
-    "text": "CSI ephemeral inline volume support is beta, i.e. the CSIInlineVolume feature gate is enabled by default",
-    "markdown": "CSI ephemeral inline volume support is beta, i.e. the CSIInlineVolume feature gate is enabled by default ([#82004](https://github.com/kubernetes/kubernetes/pull/82004), [@pohly](https://github.com/pohly))",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/20190122-csi-inline-volumes.md",
-        "type": "KEP"
-      },
-      {
-        "description": "[Usage]",
-        "url": "https://kubernetes.io/docs/concepts/storage/volumes/#csi-ephemeral-volumes",
-        "type": "official"
-      }
-    ],
-    "author": "pohly",
-    "author_url": "https://github.com/pohly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82004",
-    "pr_number": 82004,
-    "areas": ["test"],
-    "kinds": ["api-change"],
-    "sigs": ["storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "82005": {
-    "commit": "1698765538fada7fb51cd1965af028ed3953c9b0",
-    "text": "Add metadata.generation=1 to old CustomResources.",
-    "markdown": "Add metadata.generation=1 to old CustomResources. ([#82005](https://github.com/kubernetes/kubernetes/pull/82005), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82005",
-    "pr_number": 82005,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.7"
-  },
-  "82034": {
-    "commit": "b33968972bc178144b555189e56cb21eda15bce2",
-    "text": "Add UnschedulableAndUnresolvable status code for Scheduler Framework",
-    "markdown": "Add UnschedulableAndUnresolvable status code for Scheduler Framework ([#82034](https://github.com/kubernetes/kubernetes/pull/82034), [@alculquicondor](https://github.com/alculquicondor))\n\n  Courtesy of SIG Apps, SIG Scheduling, and SIG Testing",
-    "author": "alculquicondor",
-    "author_url": "https://github.com/alculquicondor",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82034",
-    "pr_number": 82034,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["apps", "scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "82090": {
-    "commit": "f442b6ef320140730f544527597a140e535f1e1d",
-    "text": "The apiserver now uses http/1.1 to communicate with admission webhooks, opening multiple connections to satisfy concurrent requests, and allowing spreading requests across multiple backing pods.",
-    "markdown": "The apiserver now uses http/1.1 to communicate with admission webhooks, opening multiple connections to satisfy concurrent requests, and allowing spreading requests across multiple backing pods. ([#82090](https://github.com/kubernetes/kubernetes/pull/82090), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82090",
-    "pr_number": 82090,
-    "areas": ["admission-control", "apiserver", "kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth", "node", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "82091": {
-    "commit": "1ade4ce2691a9323af79a50a0f1be3af1e7d9f1e",
-    "text": "Dual stack services (Phase II of IPv6DualStack feature) are enabled via the IPVS proxier. iptables proxier does not support dualstack yet. Dualstack iptables proxier is WIP and should catchup soon.\n\nto enable, kube-proxy must be have the following flags:\n--proxy-mode=ipvs \n--cluster-cidrs=\u003ccidr\u003e,\u003ccidr\u003e",
-    "markdown": "Dual stack services (Phase II of IPv6DualStack feature) are enabled via the IPVS proxier. iptables proxier does not support dualstack yet. Dualstack iptables proxier is WIP and should catchup soon.\n  \n  to enable, kube-proxy must be have the following flags:\n  --proxy-mode=ipvs \n  --cluster-cidrs=\u003ccidr\u003e,\u003ccidr\u003e ([#82091](https://github.com/kubernetes/kubernetes/pull/82091), [@khenidak](https://github.com/khenidak))",
-    "author": "khenidak",
-    "author_url": "https://github.com/khenidak",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82091",
-    "pr_number": 82091,
-    "areas": ["apiserver", "ipvs", "kubectl", "release-eng", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "apps", "cli", "network", "release", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "82110": {
-    "commit": "96a1558ab810064909f8a17685a55202b1f83575",
-    "text": "Graduating Windows GMSA support from alpha to beta",
-    "markdown": "Graduating Windows GMSA support from alpha to beta ([#82110](https://github.com/kubernetes/kubernetes/pull/82110), [@wk8](https://github.com/wk8))\n\n  Courtesy of SIG Windows",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-windows/20181221-windows-group-managed-service-accounts-for-container-identity.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "wk8",
-    "author_url": "https://github.com/wk8",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82110",
-    "pr_number": 82110,
-    "kinds": ["feature"],
-    "sigs": ["windows"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "82119": {
-    "commit": "82f5531df3908640a459793622ec0bd9b5bc8ea4",
-    "text": "Take the context as the first argument of Schedule.",
-    "markdown": "Take the context as the first argument of Schedule. ([#82119](https://github.com/kubernetes/kubernetes/pull/82119), [@wgliang](https://github.com/wgliang))",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82119",
-    "pr_number": 82119,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.7"
-  },
-  "82120": {
-    "commit": "4cc80bac919dcbb6849f31fce2cd541eb0cae002",
-    "text": "Make kubectl get --ignore-not-found continue processing when encountering error.",
-    "markdown": "Make kubectl get --ignore-not-found continue processing when encountering error. ([#82120](https://github.com/kubernetes/kubernetes/pull/82120), [@soltysh](https://github.com/soltysh))",
-    "author": "soltysh",
-    "author_url": "https://github.com/soltysh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82120",
-    "pr_number": 82120,
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.7"
-  },
-  "82127": {
-    "commit": "235f3ab3d920096097b213fa669508e78e31ec3e",
-    "text": "Kubeadm now includes CoreDNS version 1.6.2\n    - The CoreDNS Deployment now checks readiness via the `ready` plugin.\n    - The `proxy` plugin has been deprecated. The `forward` plugin is to be used instead.\n    - `kubernetes` plugin removes the `resyncperiod` option.\n    - The `upstream` option is deprecated and ignored if included.",
-    "markdown": "Kubeadm now includes CoreDNS version 1.6.2\n      - The CoreDNS Deployment now checks readiness via the `ready` plugin.\n      - The `proxy` plugin has been deprecated. The `forward` plugin is to be used instead.\n      - `kubernetes` plugin removes the `resyncperiod` option.\n      - The `upstream` option is deprecated and ignored if included. ([#82127](https://github.com/kubernetes/kubernetes/pull/82127), [@rajansandeep](https://github.com/rajansandeep))",
-    "author": "rajansandeep",
-    "author_url": "https://github.com/rajansandeep",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82127",
-    "pr_number": 82127,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.7"
-  },
-  "82140": {
-    "commit": "0466cb6e693e2846a987fe82f930f93e6e4d2a4e",
-    "text": "Node-Problem-Detector v0.7.1 is used for addon daemonset.",
-    "markdown": "Node-Problem-Detector v0.7.1 is used for addon daemonset. ([#82140](https://github.com/kubernetes/kubernetes/pull/82140), [@wangzhen127](https://github.com/wangzhen127))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Node",
-    "author": "wangzhen127",
-    "author_url": "https://github.com/wangzhen127",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82140",
-    "pr_number": 82140,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "node"],
-    "feature": true,
-    "release_version": "1.14.7"
-  },
-  "82143": {
-    "commit": "35e0f17a33e4f63b4e6328d0e0e3aabf8b6a68a6",
-    "text": "`kubectl cp` no longer supports copying symbolic links from containers; to support this use case, see `kubectl exec --help` for examples using `tar` directly",
-    "markdown": "`kubectl cp` no longer supports copying symbolic links from containers; to support this use case, see `kubectl exec --help` for examples using `tar` directly ([#82143](https://github.com/kubernetes/kubernetes/pull/82143), [@soltysh](https://github.com/soltysh))",
-    "author": "soltysh",
-    "author_url": "https://github.com/soltysh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82143",
-    "pr_number": 82143,
-    "areas": ["kubectl", "security"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.14.7"
-  },
-  "82146": {
-    "commit": "a7ac8d4c0b7fe90d51fb911511cbf6939655c877",
-    "text": "aggregated discovery requests can now timeout.  Aggregated apiserver *must- complete discovery calls within five seconds.  Other requests can take longer.\nUse feature gate `EnableAggregatedDiscoveryTimeout=false` if you *must- remove this check, but that feature gate will be removed next release.",
-    "markdown": "aggregated discovery requests can now timeout.  Aggregated apiserver *must- complete discovery calls within five seconds.  Other requests can take longer.\n  Use feature gate `EnableAggregatedDiscoveryTimeout=false` if you *must- remove this check, but that feature gate will be removed next release. ([#82146](https://github.com/kubernetes/kubernetes/pull/82146), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82146",
-    "pr_number": 82146,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.7"
-  },
-  "82170": {
-    "commit": "e3b5062057073c5800739f13a5db73073331cbd1",
-    "text": "Adding TerminationGracePeriodSeconds to the test framework API",
-    "markdown": "Adding TerminationGracePeriodSeconds to the test framework API ([#82170](https://github.com/kubernetes/kubernetes/pull/82170), [@vivekbagade](https://github.com/vivekbagade))",
-    "author": "vivekbagade",
-    "author_url": "https://github.com/vivekbagade",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82170",
-    "pr_number": 82170,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["testing"],
-    "release_version": "1.14.7"
-  },
-  "82199": {
-    "commit": "c4c64673d7746e85dd4317b05c459e81b9c49df6",
-    "text": "Updated default etcd service used by kubernetes to 3.3.15",
-    "markdown": "Updated default etcd service used by kubernetes to 3.3.15 ([#82199](https://github.com/kubernetes/kubernetes/pull/82199), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82199",
-    "pr_number": 82199,
-    "areas": ["e2e-test-framework", "kubeadm", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "82214": {
-    "commit": "f71cfdf66d0802e0f58607978f33924ffd928e8c",
-    "text": "Ensure the KUBE-MARK-DROP chain in kube-proxy mode=ipvs. The chain is ensured for both ipv4 and ipv6 in dual-stack operation.",
-    "markdown": "Ensure the KUBE-MARK-DROP chain in kube-proxy mode=ipvs. The chain is ensured for both ipv4 and ipv6 in dual-stack operation. ([#82214](https://github.com/kubernetes/kubernetes/pull/82214), [@uablrek](https://github.com/uablrek))",
-    "author": "uablrek",
-    "author_url": "https://github.com/uablrek",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82214",
-    "pr_number": 82214,
-    "areas": ["ipvs"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.7"
-  },
-  "82248": {
-    "commit": "c8c1aeaa5c5700f1e87524108896aa8c7f90f41a",
-    "text": "kubeadm: Allow users to skip the kube-proxy init addon phase during init and still be able to join a cluster and perform some other minor operations (but not upgrade).",
-    "markdown": "kubeadm: Allow users to skip the kube-proxy init addon phase during init and still be able to join a cluster and perform some other minor operations (but not upgrade). ([#82248](https://github.com/kubernetes/kubernetes/pull/82248), [@rosti](https://github.com/rosti))",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82248",
-    "pr_number": 82248,
-    "areas": ["kubeadm"],
-    "kinds": ["design"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.7"
-  },
-  "82262": {
-    "commit": "35cf6b6cbc7c55258ac0e3af51f6b0f5119750b1",
-    "text": "None.",
-    "markdown": "None. ([#82262](https://github.com/kubernetes/kubernetes/pull/82262), [@nilo19](https://github.com/nilo19))",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82262",
-    "pr_number": 82262,
-    "areas": ["provider/azure", "test"],
-    "kinds": ["bug", "failing-test"],
-    "sigs": ["testing"],
-    "release_version": "1.14.7"
-  },
-  "82267": {
-    "commit": "975d0736b35d966c50383f794c0114a34fbcd9e4",
-    "text": "kubeadm: fix for HTTPProxy check for IPv6 addresses (kubernetes/kubeadm\u0026#35;1769)",
-    "markdown": "kubeadm: fix for HTTPProxy check for IPv6 addresses (kubernetes/kubeadm\u0026#35;1769) ([#82267](https://github.com/kubernetes/kubernetes/pull/82267), [@kad](https://github.com/kad))",
-    "author": "kad",
-    "author_url": "https://github.com/kad",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82267",
-    "pr_number": 82267,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.7"
-  },
-  "82303": {
-    "commit": "26a381b8e1cf30021b0169721a874e77d6681b9c",
-    "text": "Fix a bug in apiserver that could cause a valid update request to be rejected with a precondition check failure.",
-    "markdown": "Fix a bug in apiserver that could cause a valid update request to be rejected with a precondition check failure. ([#82303](https://github.com/kubernetes/kubernetes/pull/82303), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82303",
-    "pr_number": 82303,
-    "areas": ["apiserver", "custom-resources"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.7"
-  },
-  "82314": {
-    "commit": "397ed0e8252f93484ab526b4e387f1fb4e9fc222",
-    "text": "Fix the dns suffix search list for GCE window clusters.",
-    "markdown": "Fix the dns suffix search list for GCE window clusters. ([#82314](https://github.com/kubernetes/kubernetes/pull/82314), [@lzang](https://github.com/lzang))",
-    "author": "lzang",
-    "author_url": "https://github.com/lzang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82314",
-    "pr_number": 82314,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "82322": {
-    "commit": "838e953e943dc03b05da541304733e354d29770b",
-    "text": "Bump metrics-server to v0.3.4",
-    "markdown": "Bump metrics-server to v0.3.4 ([#82322](https://github.com/kubernetes/kubernetes/pull/82322), [@olagacek](https://github.com/olagacek))",
-    "author": "olagacek",
-    "author_url": "https://github.com/olagacek",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82322",
-    "pr_number": 82322,
-    "kinds": ["bug"],
-    "sigs": ["instrumentation"],
-    "release_version": "1.14.7"
-  },
-  "82438": {
-    "commit": "e682310dcc5d805a408e0073e251d99b8fe5c06d",
-    "text": "Resolved regression serving custom resources with unhandled validation schemas with the ServerSideApply feature enabled",
-    "markdown": "Resolved regression serving custom resources with unhandled validation schemas with the ServerSideApply feature enabled ([#82438](https://github.com/kubernetes/kubernetes/pull/82438), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82438",
-    "pr_number": 82438,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "82483": {
-    "commit": "3fd8962bd1a77a3fab6b9a1bf3d5888377590d66",
-    "text": "fixed an issue that the correct PluginConfig.Args is not passed to the corresponding PluginFactory in kube-scheduler when multiple PluginConfig items are defined.",
-    "markdown": "fixed an issue that the correct PluginConfig.Args is not passed to the corresponding PluginFactory in kube-scheduler when multiple PluginConfig items are defined. ([#82483](https://github.com/kubernetes/kubernetes/pull/82483), [@everpeace](https://github.com/everpeace))",
-    "author": "everpeace",
-    "author_url": "https://github.com/everpeace",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82483",
-    "pr_number": 82483,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.7"
-  },
-  "82501": {
-    "commit": "6348200c92dec8848e55552f3e8039b3da95bd91",
-    "text": "Update Cluster Autoscaler to 1.16.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.16.0",
-    "markdown": "Update Cluster Autoscaler to 1.16.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.16.0 ([#82501](https://github.com/kubernetes/kubernetes/pull/82501), [@losipiuk](https://github.com/losipiuk))",
-    "author": "losipiuk",
-    "author_url": "https://github.com/losipiuk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82501",
-    "pr_number": 82501,
-    "kinds": ["bug"],
-    "sigs": ["autoscaling", "cluster-lifecycle"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "82514": {
-    "commit": "579e0c74c150085b3fac01f6a33b66db96922f93",
-    "text": "Limit the body length of exec readiness/liveness probes. remote CRIs and Docker shim read a max of 16MB output of which the exec probe itself inspects 10kb.",
-    "markdown": "Limit the body length of exec readiness/liveness probes. remote CRIs and Docker shim read a max of 16MB output of which the exec probe itself inspects 10kb. ([#82514](https://github.com/kubernetes/kubernetes/pull/82514), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82514",
-    "pr_number": 82514,
-    "areas": ["kubelet"],
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.14.7"
-  },
-  "82535": {
-    "commit": "1f2e9cfe4eb8949836886e7273e6266756617537",
-    "text": "Correct a reference to a not/no longer used kustomize subcommand in the documentation",
-    "markdown": "Correct a reference to a not/no longer used kustomize subcommand in the documentation ([#82535](https://github.com/kubernetes/kubernetes/pull/82535), [@demobox](https://github.com/demobox))",
-    "author": "demobox",
-    "author_url": "https://github.com/demobox",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82535",
-    "pr_number": 82535,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.14.7"
-  },
-  "82588": {
-    "commit": "17246b8026cebac663866e02a0fa6cdc0f766c90",
-    "text": "Fixes regression in logging spurious stack traces when proxied connections are closed by the backend",
-    "markdown": "Fixes regression in logging spurious stack traces when proxied connections are closed by the backend ([#82588](https://github.com/kubernetes/kubernetes/pull/82588), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82588",
-    "pr_number": 82588,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.7"
-  },
-  "82653": {
-    "commit": "21db9c2670984e0e4ed403a514309a53b248e3f6",
-    "text": "Restores compatibility with \u003c=1.15.x custom resources by not publishing OpenAPI for non-structural custom resource definitions",
-    "markdown": "Restores compatibility with \u003c=1.15.x custom resources by not publishing OpenAPI for non-structural custom resource definitions ([#82653](https://github.com/kubernetes/kubernetes/pull/82653), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82653",
-    "pr_number": 82653,
-    "areas": ["custom-resources", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.7"
-  },
-  "82669": {
-    "commit": "40695b0f48b14b1b023cb063a33a1805a319fedd",
-    "text": "Single static pod files and pod files from http endpoints cannot be larger than 10 MB. HTTP probe payloads are now truncated to 10KB.",
-    "markdown": "Single static pod files and pod files from http endpoints cannot be larger than 10 MB. HTTP probe payloads are now truncated to 10KB. ([#82669](https://github.com/kubernetes/kubernetes/pull/82669), [@rphillips](https://github.com/rphillips))",
-    "author": "rphillips",
-    "author_url": "https://github.com/rphillips",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82669",
-    "pr_number": 82669,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.7"
-  }
-}
diff --git a/src/assets/release-notes-1.14.8.json b/src/assets/release-notes-1.14.8.json
deleted file mode 100644
index 5e4aa533..00000000
--- a/src/assets/release-notes-1.14.8.json
+++ /dev/null
@@ -1,1146 +0,0 @@
-{
-  "71653": {
-    "commit": "605687dec7ae3842641930fc451d43876f5424f7",
-    "text": "The docker container runtime now enforces a 220 second timeout on container network operations.",
-    "markdown": "The docker container runtime now enforces a 220 second timeout on container network operations. ([#71653](https://github.com/kubernetes/kubernetes/pull/71653), [@liucimin](https://github.com/liucimin))",
-    "author": "liucimin",
-    "author_url": "https://github.com/liucimin",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/71653",
-    "pr_number": 71653,
-    "areas": ["dependency", "kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["network", "node"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "72170": {
-    "commit": "54d49bd785ba6fe8eb651e8a73ce3580f8c0ec41",
-    "text": "The resource version option, when passed to a list call, is now consistently interpreted as the minimum allowed resource version.  Previously when listing resources that had the watch cache disabled clients could retrieve a snapshot at that exact resource version.  If the client requests a resource version newer than the current state, a TimeoutError is returned suggesting the client retry in a few seconds.  This behavior is now consistent for both single item retrieval and list calls, and for when the watch cache is enabled or disabled.",
-    "markdown": "The resource version option, when passed to a list call, is now consistently interpreted as the minimum allowed resource version.  Previously when listing resources that had the watch cache disabled clients could retrieve a snapshot at that exact resource version.  If the client requests a resource version newer than the current state, a TimeoutError is returned suggesting the client retry in a few seconds.  This behavior is now consistent for both single item retrieval and list calls, and for when the watch cache is enabled or disabled. ([#72170](https://github.com/kubernetes/kubernetes/pull/72170), [@jpbetz](https://github.com/jpbetz))",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72170",
-    "pr_number": 72170,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.8"
-  },
-  "78764": {
-    "commit": "a7e0b768ee7e87b3dd8106b482c89e51bb2518a2",
-    "text": "kubeadm: implemented structured output of 'kubeadm token list' in JSON, YAML, Go template and JsonPath formats",
-    "markdown": "kubeadm: implemented structured output of 'kubeadm token list' in JSON, YAML, Go template and JsonPath formats ([#78764](https://github.com/kubernetes/kubernetes/pull/78764), [@bart0sh](https://github.com/bart0sh))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "bart0sh",
-    "author_url": "https://github.com/bart0sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78764",
-    "pr_number": 78764,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "80458": {
-    "commit": "2b4dc315a352a0677e1ced03557cd03087cfad68",
-    "text": "Fixes the bug in informer-gen that it produces incorrect code if a type has nonNamespaced tag set.",
-    "markdown": "Fixes the bug in informer-gen that it produces incorrect code if a type has nonNamespaced tag set. ([#80458](https://github.com/kubernetes/kubernetes/pull/80458), [@tatsuhiro-t](https://github.com/tatsuhiro-t))",
-    "author": "tatsuhiro-t",
-    "author_url": "https://github.com/tatsuhiro-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80458",
-    "pr_number": 80458,
-    "areas": ["code-generation"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.8"
-  },
-  "80465": {
-    "commit": "45f7f70479424a04029e4c437cff1d54385d6c39",
-    "text": "Resolves bottleneck in internal API server communication that can cause increased goroutines and degrade API Server performance",
-    "markdown": "Resolves bottleneck in internal API server communication that can cause increased goroutines and degrade API Server performance ([#80465](https://github.com/kubernetes/kubernetes/pull/80465), [@answer1991](https://github.com/answer1991))",
-    "author": "answer1991",
-    "author_url": "https://github.com/answer1991",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80465",
-    "pr_number": 80465,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.8"
-  },
-  "80747": {
-    "commit": "9f036cd43d35a9c41d7ac4ca82398a6d0bef957b",
-    "text": "On AWS nodes with multiple network interfaces, kubelet should now more reliably report the same primary node IP.",
-    "markdown": "On AWS nodes with multiple network interfaces, kubelet should now more reliably report the same primary node IP. ([#80747](https://github.com/kubernetes/kubernetes/pull/80747), [@danwinship](https://github.com/danwinship))",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80747",
-    "pr_number": 80747,
-    "areas": ["cloudprovider", "provider/aws"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "network", "node"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "80880": {
-    "commit": "1e94c12d416dfb1dcb4bd06631298f592abcadfa",
-    "text": "Fix validation message to mention bytes, not characters.",
-    "markdown": "Fix validation message to mention bytes, not characters. ([#80880](https://github.com/kubernetes/kubernetes/pull/80880), [@DirectXMan12](https://github.com/DirectXMan12))",
-    "author": "DirectXMan12",
-    "author_url": "https://github.com/DirectXMan12",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80880",
-    "pr_number": 80880,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "80883": {
-    "commit": "3edbc6afff17ea8dfe5c10b2677dcdc8767f67e2",
-    "text": "The mutating and validating admission webhook plugins now read configuration from the admissionregistration.k8s.io/v1 API.",
-    "markdown": "The mutating and validating admission webhook plugins now read configuration from the admissionregistration.k8s.io/v1 API. ([#80883](https://github.com/kubernetes/kubernetes/pull/80883), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80883",
-    "pr_number": 80883,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "81158": {
-    "commit": "1f5d329b4827919937a20e6fec1302668b68ef8e",
-    "text": "Fix unsafe JSON construction in a number of locations in the codebase",
-    "markdown": "Fix unsafe JSON construction in a number of locations in the codebase ([#81158](https://github.com/kubernetes/kubernetes/pull/81158), [@zouyee](https://github.com/zouyee))",
-    "author": "zouyee",
-    "author_url": "https://github.com/zouyee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81158",
-    "pr_number": 81158,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "apps", "node"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "81263": {
-    "commit": "b8b7c376edb62ce8be42cef3728112c5c71d9fa4",
-    "text": "Add \"podInitialBackoffDurationSeconds\" and \"podMaxBackoffDurationSeconds\" to the scheduler config API",
-    "markdown": "Add \"podInitialBackoffDurationSeconds\" and \"podMaxBackoffDurationSeconds\" to the scheduler config API ([#81263](https://github.com/kubernetes/kubernetes/pull/81263), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Apps, and SIG Scheduling",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81263",
-    "pr_number": 81263,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["apps", "scheduling"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "81280": {
-    "commit": "4cff1c3ea1b2478d9139bccb7018750a022ee5f0",
-    "text": "If container fails because ContainerCannotRun, do not utilize the FallbackToLogsOnError TerminationMessagePolicy, as it masks more useful logs.",
-    "markdown": "If container fails because ContainerCannotRun, do not utilize the FallbackToLogsOnError TerminationMessagePolicy, as it masks more useful logs. ([#81280](https://github.com/kubernetes/kubernetes/pull/81280), [@yqwang-ms](https://github.com/yqwang-ms))",
-    "author": "yqwang-ms",
-    "author_url": "https://github.com/yqwang-ms",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81280",
-    "pr_number": 81280,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.8"
-  },
-  "81517": {
-    "commit": "e7090e8f5e0decfc03899330824386c69af274d4",
-    "text": "Kubernetes no longer monitors firewalld. On systems using firewalld for firewall\nmaintenance, kube-proxy will take slightly longer to recover from disruptive\nfirewalld operations that delete kube-proxy's iptables rules.\n\nAs a side effect of these changes, kube-proxy's\n`sync_proxy_rules_last_timestamp_seconds` metric no longer behaves the\nway it used to; now it will only change when services or endpoints actually\nchange, rather than reliably updating every 60 seconds (or whatever). If you\nare trying to monitor for whether iptables updates are failing, the\n`sync_proxy_rules_iptables_restore_failures_total` metric may be more useful.",
-    "markdown": "Kubernetes no longer monitors firewalld. On systems using firewalld for firewall\n  maintenance, kube-proxy will take slightly longer to recover from disruptive\n  firewalld operations that delete kube-proxy's iptables rules.\n  \n  As a side effect of these changes, kube-proxy's\n  `sync_proxy_rules_last_timestamp_seconds` metric no longer behaves the\n  way it used to; now it will only change when services or endpoints actually\n  change, rather than reliably updating every 60 seconds (or whatever). If you\n  are trying to monitor for whether iptables updates are failing, the\n  `sync_proxy_rules_iptables_restore_failures_total` metric may be more useful. ([#81517](https://github.com/kubernetes/kubernetes/pull/81517), [@danwinship](https://github.com/danwinship))",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81517",
-    "pr_number": 81517,
-    "areas": ["dependency", "kube-proxy", "kubeadm", "kubelet", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "network", "node", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "82148": {
-    "commit": "5675ef858ef1039bdd5d7b8a97c71fb8da5bd581",
-    "text": "k8s dockerconfigjson secrets are now compatible with docker config desktop authentication credentials files",
-    "markdown": "k8s dockerconfigjson secrets are now compatible with docker config desktop authentication credentials files ([#82148](https://github.com/kubernetes/kubernetes/pull/82148), [@bbourbie](https://github.com/bbourbie))",
-    "author": "bbourbie",
-    "author_url": "https://github.com/bbourbie",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82148",
-    "pr_number": 82148,
-    "kinds": ["bug"],
-    "sigs": ["auth", "node"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "82176": {
-    "commit": "b140b431073ae4d84ce9ef5e01a1f27058178ead",
-    "text": "more complete and accurate logging of stack backtraces in E2E failures",
-    "markdown": "more complete and accurate logging of stack backtraces in E2E failures ([#82176](https://github.com/kubernetes/kubernetes/pull/82176), [@pohly](https://github.com/pohly))",
-    "author": "pohly",
-    "author_url": "https://github.com/pohly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82176",
-    "pr_number": 82176,
-    "areas": ["apiserver", "cloudprovider", "dependency", "e2e-test-framework", "kubectl", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "82264": {
-    "commit": "8ad7e78d37fbd1ce42d75056ea9c00a7f63aefcc",
-    "text": "Openstack: Do not delete managed LB in case of security group reconciliation errors",
-    "markdown": "Openstack: Do not delete managed LB in case of security group reconciliation errors ([#82264](https://github.com/kubernetes/kubernetes/pull/82264), [@multi-io](https://github.com/multi-io))",
-    "author": "multi-io",
-    "author_url": "https://github.com/multi-io",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82264",
-    "pr_number": 82264,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.8"
-  },
-  "82347": {
-    "commit": "129a13af983fe2b6bd4429c47e5f0cde6bf0ca42",
-    "text": "kube-dns add-on: \n- All containers are now being executed under more restrictive privileges. \n- Most of the containers now run as non-root user and has the root filesystem set as read-only. \n- The remaining container running as root only has the minimum Linux capabilities it requires to run. \n- Privilege escalation has been disabled for all containers.",
-    "markdown": "kube-dns add-on: \n  - All containers are now being executed under more restrictive privileges. \n  - Most of the containers now run as non-root user and has the root filesystem set as read-only. \n  - The remaining container running as root only has the minimum Linux capabilities it requires to run. \n  - Privilege escalation has been disabled for all containers. ([#82347](https://github.com/kubernetes/kubernetes/pull/82347), [@pjbgf](https://github.com/pjbgf))",
-    "author": "pjbgf",
-    "author_url": "https://github.com/pjbgf",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82347",
-    "pr_number": 82347,
-    "areas": ["security"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "82409": {
-    "commit": "07025a5d9ed1e697e765f725b5e9023199c41ad1",
-    "text": "Added metrics 'authentication_latency_seconds' that can be used to understand the latency of authentication.",
-    "markdown": "Added metrics 'authentication_latency_seconds' that can be used to understand the latency of authentication. ([#82409](https://github.com/kubernetes/kubernetes/pull/82409), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG API Machinery, SIG Auth, and SIG Instrumentation",
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82409",
-    "pr_number": 82409,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "auth", "instrumentation"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "82410": {
-    "commit": "973629f742e600eb5762e4c57126a73c5b3cb500",
-    "text": "Resolves regression generating informers for packages whose names contain `.` characters",
-    "markdown": "Resolves regression generating informers for packages whose names contain `.` characters ([#82410](https://github.com/kubernetes/kubernetes/pull/82410), [@nikhita](https://github.com/nikhita))",
-    "author": "nikhita",
-    "author_url": "https://github.com/nikhita",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82410",
-    "pr_number": 82410,
-    "areas": ["code-generation"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.8"
-  },
-  "82423": {
-    "commit": "ea4f7853bb24a6c3fe7aeeee1aa89f8273c8db64",
-    "text": "If a bad flag is supplied to a kubectl command, only a tip to run --help is printed, instead of the usage menu.  Usage menu is printed upon running `kubectl command --help`.",
-    "markdown": "If a bad flag is supplied to a kubectl command, only a tip to run --help is printed, instead of the usage menu.  Usage menu is printed upon running `kubectl command --help`. ([#82423](https://github.com/kubernetes/kubernetes/pull/82423), [@sallyom](https://github.com/sallyom))",
-    "author": "sallyom",
-    "author_url": "https://github.com/sallyom",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82423",
-    "pr_number": 82423,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.14.8"
-  },
-  "82432": {
-    "commit": "f2ee75cfa29210d82dd50e440dcadfa0a23e6159",
-    "text": "Expose kubernetes client in the scheduling framework handle.",
-    "markdown": "Expose kubernetes client in the scheduling framework handle. ([#82432](https://github.com/kubernetes/kubernetes/pull/82432), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Scheduling",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82432",
-    "pr_number": 82432,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "82472": {
-    "commit": "3a19f1e80b172dfceb06ffe654b1e349bac53f73",
-    "text": "The deprecated feature gates `GCERegionalPersistentDisk`, `EnableAggregatedDiscoveryTimeout` and `PersistentLocalVolumes` are now unconditionally enabled and can no longer be specified in component invocations.",
-    "markdown": "The deprecated feature gates `GCERegionalPersistentDisk`, `EnableAggregatedDiscoveryTimeout` and `PersistentLocalVolumes` are now unconditionally enabled and can no longer be specified in component invocations. ([#82472](https://github.com/kubernetes/kubernetes/pull/82472), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG API Machinery, SIG Storage, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82472",
-    "pr_number": 82472,
-    "areas": ["apiserver", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "storage", "testing"],
-    "action_required": true,
-    "release_version": "1.14.8"
-  },
-  "82473": {
-    "commit": "f3981710db8c6e90567e91f4c0d68e8122418290",
-    "text": "kubeadm: use the --service-cluster-ip-range flag to init or use the ServiceSubnet field in the kubeadm config to pass a comma separated list of Service CIDRs.",
-    "markdown": "kubeadm: use the --service-cluster-ip-range flag to init or use the ServiceSubnet field in the kubeadm config to pass a comma separated list of Service CIDRs. ([#82473](https://github.com/kubernetes/kubernetes/pull/82473), [@Arvinderpal](https://github.com/Arvinderpal))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "Arvinderpal",
-    "author_url": "https://github.com/Arvinderpal",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82473",
-    "pr_number": 82473,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "82496": {
-    "commit": "29a52f524114379f3e57917cb11c1a672d5917f4",
-    "text": "Reduces the number of calls made to the Azure API when requesting the instance view of a virtual machine scale set node.",
-    "markdown": "Reduces the number of calls made to the Azure API when requesting the instance view of a virtual machine scale set node. ([#82496](https://github.com/kubernetes/kubernetes/pull/82496), [@hasheddan](https://github.com/hasheddan))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "hasheddan",
-    "author_url": "https://github.com/hasheddan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82496",
-    "pr_number": 82496,
-    "areas": ["cloudprovider", "dependency"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "82508": {
-    "commit": "cfa3e2c499d5316ccf9f837e9bc8238cd6aeaff5",
-    "text": "Fix panic in kubelet when running IPv4/IPv6 dual-stack mode with a CNI plugin",
-    "markdown": "Fix panic in kubelet when running IPv4/IPv6 dual-stack mode with a CNI plugin ([#82508](https://github.com/kubernetes/kubernetes/pull/82508), [@aanm](https://github.com/aanm))",
-    "author": "aanm",
-    "author_url": "https://github.com/aanm",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82508",
-    "pr_number": 82508,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["network", "node"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "82511": {
-    "commit": "dd5efc420b80df2fac8a550e6e05a57a11807c50",
-    "text": "Fix typos in `certificates.k8s.io/v1beta1` KeyUsage constant names: `UsageContentCommittment` becomes `UsageContentCommitment` and `UsageNetscapSGC` becomes `UsageNetscapeSGC`.",
-    "markdown": "Fix typos in `certificates.k8s.io/v1beta1` KeyUsage constant names: `UsageContentCommittment` becomes `UsageContentCommitment` and `UsageNetscapSGC` becomes `UsageNetscapeSGC`. ([#82511](https://github.com/kubernetes/kubernetes/pull/82511), [@abursavich](https://github.com/abursavich))",
-    "author": "abursavich",
-    "author_url": "https://github.com/abursavich",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82511",
-    "pr_number": 82511,
-    "areas": ["kubeadm"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["auth", "cluster-lifecycle"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "82574": {
-    "commit": "24b1e7f27f6df30969e297038200e82a473d812f",
-    "text": "Added cloud operation count metrics to azure cloud controller manager.",
-    "markdown": "Added cloud operation count metrics to azure cloud controller manager. ([#82574](https://github.com/kubernetes/kubernetes/pull/82574), [@kkmsft](https://github.com/kkmsft))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "kkmsft",
-    "author_url": "https://github.com/kkmsft",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82574",
-    "pr_number": 82574,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "82578": {
-    "commit": "cb1799bd489029f696014343892ee55084f61266",
-    "text": "The VolumeSubpathEnvExpansion feature is graduating to GA. The `VolumeSubpathEnvExpansion` feature gate is unconditionally enabled, and will be removed in v1.19.",
-    "markdown": "The VolumeSubpathEnvExpansion feature is graduating to GA. The `VolumeSubpathEnvExpansion` feature gate is unconditionally enabled, and will be removed in v1.19. ([#82578](https://github.com/kubernetes/kubernetes/pull/82578), [@kevtaylor](https://github.com/kevtaylor))",
-    "author": "kevtaylor",
-    "author_url": "https://github.com/kevtaylor",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82578",
-    "pr_number": 82578,
-    "areas": ["test"],
-    "kinds": ["api-change"],
-    "sigs": ["apps", "node", "storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "82616": {
-    "commit": "4fb75e2f0d9a36c47edcf65f89bb92f20274ee56",
-    "text": "kubeadm: when adding extra apiserver authorization-modes, the defaults \"Node,RBAC\" are no longer prepended in the resulting static Pod manifests and a full override is allowed.",
-    "markdown": "kubeadm: when adding extra apiserver authorization-modes, the defaults \"Node,RBAC\" are no longer prepended in the resulting static Pod manifests and a full override is allowed. ([#82616](https://github.com/kubernetes/kubernetes/pull/82616), [@ghouscht](https://github.com/ghouscht))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "ghouscht",
-    "author_url": "https://github.com/ghouscht",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82616",
-    "pr_number": 82616,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "action_required": true,
-    "release_version": "1.14.8"
-  },
-  "82640": {
-    "commit": "ba07527278ef2cde9c27886ec3333cfef472112a",
-    "text": "fix: azure disk detach failure if node not exists",
-    "markdown": "fix: azure disk detach failure if node not exists ([#82640](https://github.com/kubernetes/kubernetes/pull/82640), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82640",
-    "pr_number": 82640,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.8"
-  },
-  "82643": {
-    "commit": "dd8a23ca2e366ccd247179f13da372762d81b33e",
-    "text": "\u0026#35; kubectl rollout history sts/test-sts \nstatefulset.apps/test-sts \nREVISION\n0\n0\n0\n1\n2\n3",
-    "markdown": "\u0026#35; kubectl rollout history sts/test-sts \n  statefulset.apps/test-sts \n  REVISION\n  0\n  0\n  0\n  1\n  2\n  3 ([#82643](https://github.com/kubernetes/kubernetes/pull/82643), [@ZP-AlwaysWin](https://github.com/ZP-AlwaysWin))",
-    "author": "ZP-AlwaysWin",
-    "author_url": "https://github.com/ZP-AlwaysWin",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82643",
-    "pr_number": 82643,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.8"
-  },
-  "82664": {
-    "commit": "4568c615269860433f914e1dc74b13e66acbe5ef",
-    "text": "Conformance tests may now include disruptive tests. If you are running tests against a live cluster, consider skipping those tests tagged as `Disruptive` to avoid non-test workloads being impacted. Be aware, skipping any conformance tests (even disruptive ones) will make the results ineligible for consideration for the CNCF Certified Kubernetes program.",
-    "markdown": "Conformance tests may now include disruptive tests. If you are running tests against a live cluster, consider skipping those tests tagged as `Disruptive` to avoid non-test workloads being impacted. Be aware, skipping any conformance tests (even disruptive ones) will make the results ineligible for consideration for the CNCF Certified Kubernetes program. ([#82664](https://github.com/kubernetes/kubernetes/pull/82664), [@johnSchnake](https://github.com/johnSchnake))",
-    "author": "johnSchnake",
-    "author_url": "https://github.com/johnSchnake",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82664",
-    "pr_number": 82664,
-    "areas": ["conformance", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["architecture", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "82707": {
-    "commit": "3ef3b6a5baf9e57bfda739dcb06cb0e1c882fb98",
-    "text": "- When registering with a 1.17+ API server, MutatingWebhookConfiguration and ValidatingWebhookConfiguration objects can now request that only `v1` AdmissionReview requests be sent to them. Previously, webhooks were required to support receiving `v1beta1` AdmissionReview requests as well for compatibility with API servers \u003c= 1.15.\n- When registering with a 1.17+ API server, a CustomResourceDefinition conversion webhook can now request that only `v1` ConversionReview requests be sent to them. Previously, conversion webhooks were required to support receiving `v1beta1` ConversionReview requests as well for compatibility with API servers \u003c= 1.15.",
-    "markdown": "- When registering with a 1.17+ API server, MutatingWebhookConfiguration and ValidatingWebhookConfiguration objects can now request that only `v1` AdmissionReview requests be sent to them. Previously, webhooks were required to support receiving `v1beta1` AdmissionReview requests as well for compatibility with API servers \u003c= 1.15.\n  - When registering with a 1.17+ API server, a CustomResourceDefinition conversion webhook can now request that only `v1` ConversionReview requests be sent to them. Previously, conversion webhooks were required to support receiving `v1beta1` ConversionReview requests as well for compatibility with API servers \u003c= 1.15. ([#82707](https://github.com/kubernetes/kubernetes/pull/82707), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82707",
-    "pr_number": 82707,
-    "kinds": ["api-change", "cleanup", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "82713": {
-    "commit": "61affb33267391042cc07e2679cb733925ace20a",
-    "text": "Resolves issue with /readyz and /livez not including etcd and kms health checks",
-    "markdown": "Resolves issue with /readyz and /livez not including etcd and kms health checks ([#82713](https://github.com/kubernetes/kubernetes/pull/82713), [@logicalhan](https://github.com/logicalhan))",
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82713",
-    "pr_number": 82713,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.8"
-  },
-  "82759": {
-    "commit": "caeda2112aba9a054ae9c8fae03b1708d48bee5d",
-    "text": "Report non-confusing error for negative storage size in PVC spec.",
-    "markdown": "Report non-confusing error for negative storage size in PVC spec. ([#82759](https://github.com/kubernetes/kubernetes/pull/82759), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82759",
-    "pr_number": 82759,
-    "kinds": ["bug"],
-    "sigs": ["apps", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "82830": {
-    "commit": "ac8ac0fc170eec7cdf292d841f8d026a65ce1681",
-    "text": "PersistentVolumeLabel admission plugin, responsible for labeling `PersistentVolumes` with topology labels, now does not overwrite existing labels on PVs that were dynamically provisioned. It trusts the  dynamic provisioning that it provided the correct labels to the `PersistentVolume`, saving one potentially expensive cloud API call. `PersistentVolumes` created manually by users are labelled by the admission plugin in the same way as before.",
-    "markdown": "PersistentVolumeLabel admission plugin, responsible for labeling `PersistentVolumes` with topology labels, now does not overwrite existing labels on PVs that were dynamically provisioned. It trusts the  dynamic provisioning that it provided the correct labels to the `PersistentVolume`, saving one potentially expensive cloud API call. `PersistentVolumes` created manually by users are labelled by the admission plugin in the same way as before. ([#82830](https://github.com/kubernetes/kubernetes/pull/82830), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82830",
-    "pr_number": 82830,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.8"
-  },
-  "82841": {
-    "commit": "33adc7fafa68ad3b91c55dfa4e3db0ce807ad5ab",
-    "text": "Fixed a scheduler panic when using PodAffinity.",
-    "markdown": "Fixed a scheduler panic when using PodAffinity. ([#82841](https://github.com/kubernetes/kubernetes/pull/82841), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82841",
-    "pr_number": 82841,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.8"
-  },
-  "82842": {
-    "commit": "b0e5b9c7b06b56c5d444c934ab7503d10f4d5a1b",
-    "text": "Modified the scheduling framework's Filter API.",
-    "markdown": "Modified the scheduling framework's Filter API. ([#82842](https://github.com/kubernetes/kubernetes/pull/82842), [@ahg-g](https://github.com/ahg-g))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "ahg-g",
-    "author_url": "https://github.com/ahg-g",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82842",
-    "pr_number": 82842,
-    "areas": ["test"],
-    "kinds": ["bug", "feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "82856": {
-    "commit": "b281315450d1882aeb9ccf2ecdc888626b89b50a",
-    "text": "Update crictl to v1.16.1.",
-    "markdown": "Update crictl to v1.16.1. ([#82856](https://github.com/kubernetes/kubernetes/pull/82856), [@Random-Liu](https://github.com/Random-Liu))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Node",
-    "author": "Random-Liu",
-    "author_url": "https://github.com/Random-Liu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82856",
-    "pr_number": 82856,
-    "kinds": ["bug", "feature"],
-    "sigs": ["cluster-lifecycle", "node"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "82873": {
-    "commit": "338a09abd26a7b549cb9ba17655f9fe1c0c10f44",
-    "text": "Fix possible fd leak and closing of dirs when using openstack",
-    "markdown": "Fix possible fd leak and closing of dirs when using openstack ([#82873](https://github.com/kubernetes/kubernetes/pull/82873), [@odinuge](https://github.com/odinuge))",
-    "author": "odinuge",
-    "author_url": "https://github.com/odinuge",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82873",
-    "pr_number": 82873,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "82887": {
-    "commit": "73505056fb757140da6616d61e905222ef4fe939",
-    "text": "Fixes a panic in kube-controller-manager cleaning up bootstrap tokens",
-    "markdown": "Fixes a panic in kube-controller-manager cleaning up bootstrap tokens ([#82887](https://github.com/kubernetes/kubernetes/pull/82887), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82887",
-    "pr_number": 82887,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.8"
-  },
-  "82890": {
-    "commit": "e89023cff364a4af92a0abcfdfbf47ca7819d1a5",
-    "text": "Update Azure load balancer to prevent orphaned public IP addresses",
-    "markdown": "Update Azure load balancer to prevent orphaned public IP addresses ([#82890](https://github.com/kubernetes/kubernetes/pull/82890), [@chewong](https://github.com/chewong))",
-    "author": "chewong",
-    "author_url": "https://github.com/chewong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82890",
-    "pr_number": 82890,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "cloud-provider", "network"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "82912": {
-    "commit": "0b4cccc9d12af5a097b0d13f101566cbcfd9895f",
-    "text": "New APIs to allow adding/removing pods from pre-calculated prefilter state in the scheduling framework",
-    "markdown": "New APIs to allow adding/removing pods from pre-calculated prefilter state in the scheduling framework ([#82912](https://github.com/kubernetes/kubernetes/pull/82912), [@ahg-g](https://github.com/ahg-g))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "ahg-g",
-    "author_url": "https://github.com/ahg-g",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82912",
-    "pr_number": 82912,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "82927": {
-    "commit": "0ff761b14f26dc59815f63c72dc00d2f69f4f7b8",
-    "text": "kube-proxy: a configuration file specified via `--config` is now loaded with strict deserialization, which fails if the config file contains duplicate or unknown fields. This protects against accidentally running with config files that are malformed, mis-indented, or have typos in field names, and getting unexpected behavior.",
-    "markdown": "kube-proxy: a configuration file specified via `--config` is now loaded with strict deserialization, which fails if the config file contains duplicate or unknown fields. This protects against accidentally running with config files that are malformed, mis-indented, or have typos in field names, and getting unexpected behavior. ([#82927](https://github.com/kubernetes/kubernetes/pull/82927), [@obitech](https://github.com/obitech))\n\n  Courtesy of SIG API Machinery, SIG Cluster Lifecycle, and SIG Network",
-    "author": "obitech",
-    "author_url": "https://github.com/obitech",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82927",
-    "pr_number": 82927,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "network"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "82951": {
-    "commit": "05aaeac8b2b4a4c9a625c11648fc72e5cab07f84",
-    "text": "Added Clone method to the scheduling framework's PluginContext and ContextData.",
-    "markdown": "Added Clone method to the scheduling framework's PluginContext and ContextData. ([#82951](https://github.com/kubernetes/kubernetes/pull/82951), [@ahg-g](https://github.com/ahg-g))\n\n  Courtesy of SIG Scheduling",
-    "author": "ahg-g",
-    "author_url": "https://github.com/ahg-g",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82951",
-    "pr_number": 82951,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "82975": {
-    "commit": "6f5c2507ea53612e411d550d62e2ebc6cd4a9767",
-    "text": "dashboard: disable the dashboard Deployment on non-Linux nodes. This step is required to support Windows worker nodes.",
-    "markdown": "dashboard: disable the dashboard Deployment on non-Linux nodes. This step is required to support Windows worker nodes. ([#82975](https://github.com/kubernetes/kubernetes/pull/82975), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82975",
-    "pr_number": 82975,
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.8"
-  },
-  "82996": {
-    "commit": "b00f00931619864a9dba9ffcf0f5d0b24b073aa2",
-    "text": "Fixed the bug that deleted services were processed by EndpointSliceController repeatedly even their cleanup were successful.",
-    "markdown": "Fixed the bug that deleted services were processed by EndpointSliceController repeatedly even their cleanup were successful. ([#82996](https://github.com/kubernetes/kubernetes/pull/82996), [@tnqn](https://github.com/tnqn))",
-    "author": "tnqn",
-    "author_url": "https://github.com/tnqn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82996",
-    "pr_number": 82996,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.14.8"
-  },
-  "83015": {
-    "commit": "9b376e4e8e841c36667873d1ffb3f9b1abfed315",
-    "text": "Bump metrics-server to v0.3.5",
-    "markdown": "Bump metrics-server to v0.3.5 ([#83015](https://github.com/kubernetes/kubernetes/pull/83015), [@olagacek](https://github.com/olagacek))",
-    "author": "olagacek",
-    "author_url": "https://github.com/olagacek",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83015",
-    "pr_number": 83015,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.8"
-  },
-  "83027": {
-    "commit": "14d889f6efbee65671e8de7ff02ea02ebda5347c",
-    "text": "Fixes kube-proxy bug accessing self nodeip:port on windows",
-    "markdown": "Fixes kube-proxy bug accessing self nodeip:port on windows ([#83027](https://github.com/kubernetes/kubernetes/pull/83027), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83027",
-    "pr_number": 83027,
-    "areas": ["dependency"],
-    "kinds": ["bug"],
-    "sigs": ["network", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "83030": {
-    "commit": "6dde8daa767d0d51cb3c93a4ff966498c5210a13",
-    "text": "kube-scheduler: a configuration file specified via `--config` is now loaded with strict deserialization, which fails if the config file contains duplicate or unknown fields. This protects against accidentally running with config files that are malformed, mis-indented, or have typos in field names, and getting unexpected behavior.",
-    "markdown": "kube-scheduler: a configuration file specified via `--config` is now loaded with strict deserialization, which fails if the config file contains duplicate or unknown fields. This protects against accidentally running with config files that are malformed, mis-indented, or have typos in field names, and getting unexpected behavior. ([#83030](https://github.com/kubernetes/kubernetes/pull/83030), [@obitech](https://github.com/obitech))\n\n  Courtesy of SIG API Machinery, SIG Cluster Lifecycle, and SIG Scheduling",
-    "author": "obitech",
-    "author_url": "https://github.com/obitech",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83030",
-    "pr_number": 83030,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "scheduling"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "83035": {
-    "commit": "7de6f0eb95b6b35ed0426622fc0fc30543ac6c3e",
-    "text": "Improved performance of kube-proxy with EndpointSlice enabled with more efficient sorting.",
-    "markdown": "Improved performance of kube-proxy with EndpointSlice enabled with more efficient sorting. ([#83035](https://github.com/kubernetes/kubernetes/pull/83035), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83035",
-    "pr_number": 83035,
-    "kinds": ["cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.14.8"
-  },
-  "83036": {
-    "commit": "f243640e222913d122a0dee4a47ca4c1609e6123",
-    "text": "Use ipv4 in wincat port forward.",
-    "markdown": "Use ipv4 in wincat port forward. ([#83036](https://github.com/kubernetes/kubernetes/pull/83036), [@liyanhui1228](https://github.com/liyanhui1228))",
-    "author": "liyanhui1228",
-    "author_url": "https://github.com/liyanhui1228",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83036",
-    "pr_number": 83036,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.8"
-  },
-  "83042": {
-    "commit": "5e4abfb94613339d585e5d811e4fcec278ff06dc",
-    "text": "Consolidate ScoreWithNormalizePlugin into the ScorePlugin interface",
-    "markdown": "Consolidate ScoreWithNormalizePlugin into the ScorePlugin interface ([#83042](https://github.com/kubernetes/kubernetes/pull/83042), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83042",
-    "pr_number": 83042,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "83093": {
-    "commit": "1bfcfa68a9d5e03d9225833d9d15939e66be8593",
-    "text": "The deprecated mondo `kubernetes-test` tarball is no longer built. Users running Kubernetes e2e tests should use the `kubernetes-test-portable` and `kubernetes-test-{OS}-{ARCH}` tarballs instead.",
-    "markdown": "The deprecated mondo `kubernetes-test` tarball is no longer built. Users running Kubernetes e2e tests should use the `kubernetes-test-portable` and `kubernetes-test-{OS}-{ARCH}` tarballs instead. ([#83093](https://github.com/kubernetes/kubernetes/pull/83093), [@ixdy](https://github.com/ixdy))",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-testing/20190118-breaking-apart-the-kubernetes-test-tarball.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "ixdy",
-    "author_url": "https://github.com/ixdy",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83093",
-    "pr_number": 83093,
-    "areas": ["release-eng"],
-    "kinds": ["cleanup"],
-    "sigs": ["release", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "83102": {
-    "commit": "77d0ebe39fde51adf3aa78e9ae70cb7acc98182c",
-    "text": "Fix aggressive VM calls for Azure VMSS",
-    "markdown": "Fix aggressive VM calls for Azure VMSS ([#83102](https://github.com/kubernetes/kubernetes/pull/83102), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83102",
-    "pr_number": 83102,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.8"
-  },
-  "83104": {
-    "commit": "24424e26d1b2f898b8a26618d0414862f04f2917",
-    "text": "IP validates if a string is a valid IP address",
-    "markdown": "IP validates if a string is a valid IP address ([#83104](https://github.com/kubernetes/kubernetes/pull/83104), [@zouyee](https://github.com/zouyee))",
-    "author": "zouyee",
-    "author_url": "https://github.com/zouyee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83104",
-    "pr_number": 83104,
-    "kinds": ["flake"],
-    "sigs": ["storage"],
-    "release_version": "1.14.8"
-  },
-  "83139": {
-    "commit": "66755bda05b73305c4ed855d1c106b17f0dd3de7",
-    "text": "Update to go 1.12.10",
-    "markdown": "Update to go 1.12.10 ([#83139](https://github.com/kubernetes/kubernetes/pull/83139), [@cblecker](https://github.com/cblecker))",
-    "author": "cblecker",
-    "author_url": "https://github.com/cblecker",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83139",
-    "pr_number": 83139,
-    "areas": ["release-eng", "test"],
-    "kinds": ["bug"],
-    "sigs": ["release", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "83184": {
-    "commit": "91b2a7af8745ba610a7749ece553d06495178535",
-    "text": "Fix error where metrics related to dynamic kubelet config isn't registered",
-    "markdown": "Fix error where metrics related to dynamic kubelet config isn't registered ([#83184](https://github.com/kubernetes/kubernetes/pull/83184), [@odinuge](https://github.com/odinuge))",
-    "author": "odinuge",
-    "author_url": "https://github.com/odinuge",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83184",
-    "pr_number": 83184,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.8"
-  },
-  "83196": {
-    "commit": "be1574b741cc926a77ebde551b74cdabb6a0be7f",
-    "text": "Use online nodes instead of possible nodes when discovering available NUMA nodes",
-    "markdown": "Use online nodes instead of possible nodes when discovering available NUMA nodes ([#83196](https://github.com/kubernetes/kubernetes/pull/83196), [@zouyee](https://github.com/zouyee))",
-    "author": "zouyee",
-    "author_url": "https://github.com/zouyee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83196",
-    "pr_number": 83196,
-    "areas": ["kubelet"],
-    "kinds": ["flake"],
-    "sigs": ["node"],
-    "release_version": "1.14.8"
-  },
-  "83208": {
-    "commit": "35d68586dbe37ea2d317baabeee86a4db6d1f33f",
-    "text": "Significant kube-proxy performance improvements for non UDP ports.",
-    "markdown": "Significant kube-proxy performance improvements for non UDP ports. ([#83208](https://github.com/kubernetes/kubernetes/pull/83208), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83208",
-    "pr_number": 83208,
-    "areas": ["ipvs"],
-    "kinds": ["cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.14.8"
-  },
-  "83261": {
-    "commit": "4afcba42bed2bb7c36e5209a90d87343f32a0efa",
-    "text": "Fixes a flaw (CVE-2019-11253) in json/yaml decoding where large or malformed documents could consume excessive server resources. Request bodies for normal API requests (create/delete/update/patch operations of regular resources) are now limited to 3MB.",
-    "markdown": "Fixes a flaw (CVE-2019-11253) in json/yaml decoding where large or malformed documents could consume excessive server resources. Request bodies for normal API requests (create/delete/update/patch operations of regular resources) are now limited to 3MB. ([#83261](https://github.com/kubernetes/kubernetes/pull/83261), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83261",
-    "pr_number": 83261,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "83262": {
-    "commit": "9d59ed80f53a2828343105646f23142a31df8473",
-    "text": "Some scheduler extender API fields are moved from `pkg/scheduler/api` to `pkg/scheduler/apis/extender/v1`.",
-    "markdown": "Some scheduler extender API fields are moved from `pkg/scheduler/api` to `pkg/scheduler/apis/extender/v1`. ([#83262](https://github.com/kubernetes/kubernetes/pull/83262), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83262",
-    "pr_number": 83262,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "83276": {
-    "commit": "6564b34f9e274f62ce38ff11b54393c1f3e4d7fd",
-    "text": "`kubeadm.k8s.io/v1beta1` has been deprecated, you should update your config to use newer non-deprecated API versions.",
-    "markdown": "`kubeadm.k8s.io/v1beta1` has been deprecated, you should update your config to use newer non-deprecated API versions. ([#83276](https://github.com/kubernetes/kubernetes/pull/83276), [@Klaven](https://github.com/Klaven))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "Klaven",
-    "author_url": "https://github.com/Klaven",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83276",
-    "pr_number": 83276,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "action_required": true,
-    "release_version": "1.14.8"
-  },
-  "83333": {
-    "commit": "acd9141e76d3453c08700f22b24bdc320718d5dc",
-    "text": "Fixes a goroutine leak in kube-apiserver when a request times out.",
-    "markdown": "Fixes a goroutine leak in kube-apiserver when a request times out. ([#83333](https://github.com/kubernetes/kubernetes/pull/83333), [@lavalamp](https://github.com/lavalamp))",
-    "author": "lavalamp",
-    "author_url": "https://github.com/lavalamp",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83333",
-    "pr_number": 83333,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.8"
-  },
-  "83386": {
-    "commit": "8a123346838d4b96b68835aebf45a4334867113c",
-    "text": "Remove MaxPriority in the scheduler API, please use MaxNodeScore or MaxExtenderPriority instead.",
-    "markdown": "Remove MaxPriority in the scheduler API, please use MaxNodeScore or MaxExtenderPriority instead. ([#83386](https://github.com/kubernetes/kubernetes/pull/83386), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83386",
-    "pr_number": 83386,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "83396": {
-    "commit": "56f60d1bcc81270fd9239ddb6b8ed83b1beb77a6",
-    "text": "Bump version of event-exporter to 0.3.1, to switch it to protobuf.",
-    "markdown": "Bump version of event-exporter to 0.3.1, to switch it to protobuf. ([#83396](https://github.com/kubernetes/kubernetes/pull/83396), [@loburm](https://github.com/loburm))\n\n  Courtesy of SIG Instrumentation, and SIG Scalability",
-    "author": "loburm",
-    "author_url": "https://github.com/loburm",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83396",
-    "pr_number": 83396,
-    "kinds": ["feature"],
-    "sigs": ["instrumentation", "scalability"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "83430": {
-    "commit": "eba349b855c0ac70d4a954127cb5ed358cc0a8ab",
-    "text": "Rename PluginContext to CycleState in the scheduling framework",
-    "markdown": "Rename PluginContext to CycleState in the scheduling framework ([#83430](https://github.com/kubernetes/kubernetes/pull/83430), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83430",
-    "pr_number": 83430,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "83451": {
-    "commit": "8098bae92114a85134f30aa0a6d7bb72c37412ae",
-    "text": "Fixed cleanup of raw block devices after kubelet restart.",
-    "markdown": "Fixed cleanup of raw block devices after kubelet restart. ([#83451](https://github.com/kubernetes/kubernetes/pull/83451), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83451",
-    "pr_number": 83451,
-    "areas": ["kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["node", "storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "83454": {
-    "commit": "7c540ecdac946edfefe03bba50d1dd08ec4b62d5",
-    "text": "hyperkube will now be available in a new github repository and will not be included in the kubernetes release from 1.17 onwards",
-    "markdown": "hyperkube will now be available in a new github repository and will not be included in the kubernetes release from 1.17 onwards ([#83454](https://github.com/kubernetes/kubernetes/pull/83454), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83454",
-    "pr_number": 83454,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "release"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "83475": {
-    "commit": "5a4c6c2b34a011ca1801354395538df5b8e8cd66",
-    "text": "Kubeadm: add support for 127.0.0.1 as advertise address. kubeadm will automatically replace this value with matching global unicast IP address on the loopback interface.",
-    "markdown": "Kubeadm: add support for 127.0.0.1 as advertise address. kubeadm will automatically replace this value with matching global unicast IP address on the loopback interface. ([#83475](https://github.com/kubernetes/kubernetes/pull/83475), [@fabriziopandini](https://github.com/fabriziopandini))\n\n  Courtesy of SIG API Machinery, and SIG Cluster Lifecycle",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83475",
-    "pr_number": 83475,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "83492": {
-    "commit": "017842d49deca68dab3a26b76d5dfd5df15a8113",
-    "text": "The topology manager aligns resources for pods of all QoS classes with respect to NUMA locality, not just Guaranteed QoS pods.",
-    "markdown": "The topology manager aligns resources for pods of all QoS classes with respect to NUMA locality, not just Guaranteed QoS pods. ([#83492](https://github.com/kubernetes/kubernetes/pull/83492), [@ConnorDoyle](https://github.com/ConnorDoyle))\n\n  Courtesy of SIG Node",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/c1e2ceed191fa3b1d7bc0a1e3ae6c472083ad09b/keps/sig-node/0035-20190130-topology-manager.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "ConnorDoyle",
-    "author_url": "https://github.com/ConnorDoyle",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83492",
-    "pr_number": 83492,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "83522": {
-    "commit": "1e5a9f3ff4eef0b190c4300037607097b8027439",
-    "text": "Expand scheduler priority functions and scheduling framework plugins' node score range to [0, 100]. action required. Note: this change is internal and does not affect extender and RequestedToCapacityRatio custom priority, which are still expected to provide a [0, 10] range.",
-    "markdown": "Expand scheduler priority functions and scheduling framework plugins' node score range to [0, 100]. action required. Note: this change is internal and does not affect extender and RequestedToCapacityRatio custom priority, which are still expected to provide a [0, 10] range. ([#83522](https://github.com/kubernetes/kubernetes/pull/83522), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Scheduling",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83522",
-    "pr_number": 83522,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "action_required": true,
-    "release_version": "1.14.8"
-  },
-  "83528": {
-    "commit": "c8d4d626d8e9baaee885f85cd2476395e597ec14",
-    "text": "kubeadm: fix wrong default value for the \"upgrade node --certificate-renewal\" flag.",
-    "markdown": "kubeadm: fix wrong default value for the \"upgrade node --certificate-renewal\" flag. ([#83528](https://github.com/kubernetes/kubernetes/pull/83528), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83528",
-    "pr_number": 83528,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.8"
-  },
-  "83539": {
-    "commit": "d38f340e4ff6214b3365b05e744b965d0d32bdcf",
-    "text": "Add more tracing steps in generic_scheduler",
-    "markdown": "Add more tracing steps in generic_scheduler ([#83539](https://github.com/kubernetes/kubernetes/pull/83539), [@wgliang](https://github.com/wgliang))",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83539",
-    "pr_number": 83539,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.8"
-  },
-  "83547": {
-    "commit": "70d453d32df254ef0c3420804c6487a2ba55a5e0",
-    "text": "The `--certificate-authority` flag now correctly overrides existing skip TLS or CA data settings in the kubeconfig file",
-    "markdown": "The `--certificate-authority` flag now correctly overrides existing skip TLS or CA data settings in the kubeconfig file ([#83547](https://github.com/kubernetes/kubernetes/pull/83547), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83547",
-    "pr_number": 83547,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "83551": {
-    "commit": "3db6d3abcf11a40029fa6c8b5096535dfc422368",
-    "text": "external facing APIs in pluginregistration and deviceplugin packages are now available under k8s.io/kubelet/pkg/apis/",
-    "markdown": "external facing APIs in pluginregistration and deviceplugin packages are now available under k8s.io/kubelet/pkg/apis/ ([#83551](https://github.com/kubernetes/kubernetes/pull/83551), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83551",
-    "pr_number": 83551,
-    "areas": ["code-organization", "dependency", "kubelet", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["node", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "83552": {
-    "commit": "c42eceb1365cfee18ebf4e09083ce112abc95b3a",
-    "text": "Commands like `kubectl apply` now return errors if schema-invalid annotations are specified, rather than silently dropping the entire annotations section.",
-    "markdown": "Commands like `kubectl apply` now return errors if schema-invalid annotations are specified, rather than silently dropping the entire annotations section. ([#83552](https://github.com/kubernetes/kubernetes/pull/83552), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83552",
-    "pr_number": 83552,
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.8"
-  },
-  "83596": {
-    "commit": "6d97d36fbb5a52e9e7520fb326283a53ba1570a4",
-    "text": "Bumps the minimum version of Go required for building Kubernetes to 1.12.4.",
-    "markdown": "Bumps the minimum version of Go required for building Kubernetes to 1.12.4. ([#83596](https://github.com/kubernetes/kubernetes/pull/83596), [@jktomer](https://github.com/jktomer))",
-    "author": "jktomer",
-    "author_url": "https://github.com/jktomer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83596",
-    "pr_number": 83596,
-    "kinds": ["cleanup"],
-    "sigs": ["release"],
-    "release_version": "1.14.8"
-  },
-  "83599": {
-    "commit": "d8a3fbd27590ff4b56252dc4b2d0b54323a98b98",
-    "text": "kube-proxy iptables probabilities are now more granular and will result in better distribution beyond 319 endpoints.",
-    "markdown": "kube-proxy iptables probabilities are now more granular and will result in better distribution beyond 319 endpoints. ([#83599](https://github.com/kubernetes/kubernetes/pull/83599), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83599",
-    "pr_number": 83599,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.8"
-  },
-  "83603": {
-    "commit": "54f089fa08704f3fec01ae429452409acfbd1293",
-    "text": "Change `pod_preemption_victims` metric from Gauge to Histogram.",
-    "markdown": "Change `pod_preemption_victims` metric from Gauge to Histogram. ([#83603](https://github.com/kubernetes/kubernetes/pull/83603), [@Tabrizian](https://github.com/Tabrizian))\n\n  Courtesy of SIG Scheduling",
-    "author": "Tabrizian",
-    "author_url": "https://github.com/Tabrizian",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83603",
-    "pr_number": 83603,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "83643": {
-    "commit": "6992d1386c481437acbfc2a7edcf4ae3575cc85c",
-    "text": "Authentication token cache size is increased (from 4k to 32k) to support clusters with many nodes or many namespaces with active service accounts.",
-    "markdown": "Authentication token cache size is increased (from 4k to 32k) to support clusters with many nodes or many namespaces with active service accounts. ([#83643](https://github.com/kubernetes/kubernetes/pull/83643), [@lavalamp](https://github.com/lavalamp))",
-    "author": "lavalamp",
-    "author_url": "https://github.com/lavalamp",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83643",
-    "pr_number": 83643,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth"],
-    "duplicate": true,
-    "release_version": "1.14.8"
-  },
-  "83662": {
-    "commit": "afa394bd493304a8d940e2b74e5e71c46e972fcd",
-    "text": "[migration phase 1] PodFitsHost as filter plugin",
-    "markdown": "[migration phase 1] PodFitsHost as filter plugin ([#83662](https://github.com/kubernetes/kubernetes/pull/83662), [@wgliang](https://github.com/wgliang))",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83662",
-    "pr_number": 83662,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.8"
-  },
-  "83663": {
-    "commit": "c980000176b527aec3cf925d3be2d8f8ca546e27",
-    "text": "Expose SharedInformerFactory in the framework handle",
-    "markdown": "Expose SharedInformerFactory in the framework handle ([#83663](https://github.com/kubernetes/kubernetes/pull/83663), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Apps, SIG Scheduling, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83663",
-    "pr_number": 83663,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["apps", "scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "83674": {
-    "commit": "c41942b81dc2561deb36239828d5f99692547e02",
-    "text": "Add per-pod scheduling metrics across 1 or more schedule attempts.",
-    "markdown": "Add per-pod scheduling metrics across 1 or more schedule attempts. ([#83674](https://github.com/kubernetes/kubernetes/pull/83674), [@liu-cong](https://github.com/liu-cong))\n\n  Courtesy of SIG Scheduling",
-    "author": "liu-cong",
-    "author_url": "https://github.com/liu-cong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83674",
-    "pr_number": 83674,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.14.8"
-  },
-  "83697": {
-    "commit": "4561b67971213cad640ccb26d3c5830fff864621",
-    "text": "Fixed a bug in the single-numa-node policy of the TopologyManager.  Previously, pods that only requested CPU resources and did not request any third-party devices would fail to launch with a TopologyAffinity error. Now they will launch successfully.",
-    "markdown": "Fixed a bug in the single-numa-node policy of the TopologyManager.  Previously, pods that only requested CPU resources and did not request any third-party devices would fail to launch with a TopologyAffinity error. Now they will launch successfully. ([#83697](https://github.com/kubernetes/kubernetes/pull/83697), [@klueska](https://github.com/klueska))",
-    "author": "klueska",
-    "author_url": "https://github.com/klueska",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83697",
-    "pr_number": 83697,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.8"
-  },
-  "83787": {
-    "commit": "f86b0814a5f78b8813a6c11835285dce421f8e1e",
-    "text": "Fixed panic when accessing CustomResources of a CRD with x-kubernetes-int-or-string.",
-    "markdown": "Fixed panic when accessing CustomResources of a CRD with x-kubernetes-int-or-string. ([#83787](https://github.com/kubernetes/kubernetes/pull/83787), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83787",
-    "pr_number": 83787,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.8"
-  }
-}
diff --git a/src/assets/release-notes-1.14.9.json b/src/assets/release-notes-1.14.9.json
deleted file mode 100644
index 8b191b25..00000000
--- a/src/assets/release-notes-1.14.9.json
+++ /dev/null
@@ -1,1675 +0,0 @@
-{
-  "75588": {
-    "commit": "0a08798b9a6867ee32c5555b3a887a47ec70d10e",
-    "text": "Ceph RBD volume plugin now does not use any keyring (/etc/ceph/ceph.client.lvs01cinder.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin) for authentication. Ceph user credentials must be provided in PersistentVolume objects and referred Secrets.",
-    "markdown": "Ceph RBD volume plugin now does not use any keyring (/etc/ceph/ceph.client.lvs01cinder.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin) for authentication. Ceph user credentials must be provided in PersistentVolume objects and referred Secrets. ([#75588](https://github.com/kubernetes/kubernetes/pull/75588), [@smileusd](https://github.com/smileusd))",
-    "author": "smileusd",
-    "author_url": "https://github.com/smileusd",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75588",
-    "pr_number": 75588,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.9"
-  },
-  "76471": {
-    "commit": "39929d82d0f03416e291195e583678cbd3f3e0ad",
-    "text": "A new `--prefix` flag added into `kubectl logs` which prepends each log line with information about it's source (pod name and container name)",
-    "markdown": "A new `--prefix` flag added into `kubectl logs` which prepends each log line with information about it's source (pod name and container name) ([#76471](https://github.com/kubernetes/kubernetes/pull/76471), [@m1kola](https://github.com/m1kola))\n\n  Courtesy of SIG CLI",
-    "author": "m1kola",
-    "author_url": "https://github.com/m1kola",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76471",
-    "pr_number": 76471,
-    "areas": ["kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "80004": {
-    "commit": "e129a6bc3f39eea5cdadb4d3ed2bccd548ab8920",
-    "text": "When scaling down a ReplicaSet, delete doubled up replicas first, where a \"doubled up replica\" is defined as one that is on the same node as an active replica belonging to a related ReplicaSet.  ReplicaSets are considered \"related\" if they have a common controller (typically a Deployment).",
-    "markdown": "When scaling down a ReplicaSet, delete doubled up replicas first, where a \"doubled up replica\" is defined as one that is on the same node as an active replica belonging to a related ReplicaSet.  ReplicaSets are considered \"related\" if they have a common controller (typically a Deployment). ([#80004](https://github.com/kubernetes/kubernetes/pull/80004), [@Miciah](https://github.com/Miciah))\n\n  Courtesy of SIG Apps, SIG Autoscaling, SIG Scalability, and SIG Testing",
-    "author": "Miciah",
-    "author_url": "https://github.com/Miciah",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80004",
-    "pr_number": 80004,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["api-change", "bug", "feature"],
-    "sigs": ["apps", "autoscaling", "scalability", "testing"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "81344": {
-    "commit": "17a57f99d59c8e32d5539538523f91f97f4934a7",
-    "text": "sourcesReady provides the readiness of kubelet configuration sources such as apiserver update readiness.",
-    "markdown": "sourcesReady provides the readiness of kubelet configuration sources such as apiserver update readiness. ([#81344](https://github.com/kubernetes/kubernetes/pull/81344), [@zouyee](https://github.com/zouyee))",
-    "author": "zouyee",
-    "author_url": "https://github.com/zouyee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81344",
-    "pr_number": 81344,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "node"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "81431": {
-    "commit": "3ee195f0e9b691557d6f2c8b8b2009c6acbbf93f",
-    "text": "Deprecate the beta labels for zones (\"failure-domain.beta.kubernetes.io/zone\") and \nregions (\"failure-domain.beta.kubernetes.io/region\") in favor of their GA equivalents:\n\"topology.kubernetes.io/zone\" and \"topology.kubernetes.io/region\". \n\nThe beta labels \"failure-domain.beta.kubernetes.io/zone\" and \"failure-domain.beta.kubernetes.io/region\" will be removed in v1.21",
-    "markdown": "Deprecate the beta labels for zones (\"failure-domain.beta.kubernetes.io/zone\") and \n  regions (\"failure-domain.beta.kubernetes.io/region\") in favor of their GA equivalents:\n  \"topology.kubernetes.io/zone\" and \"topology.kubernetes.io/region\". \n  \n  The beta labels \"failure-domain.beta.kubernetes.io/zone\" and \"failure-domain.beta.kubernetes.io/region\" will be removed in v1.21 ([#81431](https://github.com/kubernetes/kubernetes/pull/81431), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81431",
-    "pr_number": 81431,
-    "areas": ["cloudprovider", "e2e-test-framework", "kubelet", "test"],
-    "kinds": ["api-change"],
-    "sigs": [
-      "apps",
-      "auth",
-      "cloud-provider",
-      "network",
-      "node",
-      "scheduling",
-      "storage",
-      "testing"
-    ],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "81670": {
-    "commit": "eb42df3e9cdb522578845c7476a3708ba7002ee0",
-    "text": "The example API server has renamed its `wardle.k8s.io` API group to `wardle.example.com`",
-    "markdown": "The example API server has renamed its `wardle.k8s.io` API group to `wardle.example.com` ([#81670](https://github.com/kubernetes/kubernetes/pull/81670), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81670",
-    "pr_number": 81670,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "81971": {
-    "commit": "3ebee370c5d65661bc59fa3c10be4919b5f94bf1",
-    "text": "The kubectl's api-resource command now has a `--sort-by` flag to sort resources by name or kind.",
-    "markdown": "The kubectl's api-resource command now has a `--sort-by` flag to sort resources by name or kind. ([#81971](https://github.com/kubernetes/kubernetes/pull/81971), [@laddng](https://github.com/laddng))\n\n  Courtesy of SIG CLI",
-    "author": "laddng",
-    "author_url": "https://github.com/laddng",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81971",
-    "pr_number": 81971,
-    "areas": ["kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "82049": {
-    "commit": "9cf309ed59bfda30d0f9279120db1121367bf5da",
-    "text": "Deprecate the instance type beta label (\"beta.kubernetes.io/instance-type\") in favor of it's GA equivalent: \"node.kubernetes.io/instance-type\"",
-    "markdown": "Deprecate the instance type beta label (\"beta.kubernetes.io/instance-type\") in favor of it's GA equivalent: \"node.kubernetes.io/instance-type\" ([#82049](https://github.com/kubernetes/kubernetes/pull/82049), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82049",
-    "pr_number": 82049,
-    "areas": ["kubelet"],
-    "kinds": ["api-change"],
-    "sigs": ["apps", "auth", "cloud-provider", "node", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "82603": {
-    "commit": "c2513c8ae30cf1d5a7d0b9a8b76c360760191b8e",
-    "text": "Fixed kubectl endpointslice output for get requests",
-    "markdown": "Fixed kubectl endpointslice output for get requests ([#82603](https://github.com/kubernetes/kubernetes/pull/82603), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82603",
-    "pr_number": 82603,
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.9"
-  },
-  "82703": {
-    "commit": "aab740ffc2444dd91ab59a034fdb490370b9b1cc",
-    "text": "Graduate TaintNodesByCondition to GA in 1.17. (feature gate will be removed in 1.18) action required",
-    "markdown": "Graduate TaintNodesByCondition to GA in 1.17. (feature gate will be removed in 1.18) action required ([#82703](https://github.com/kubernetes/kubernetes/pull/82703), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Cloud Provider, SIG Node, SIG Scheduling, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82703",
-    "pr_number": 82703,
-    "areas": ["apiserver", "kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "apps", "cloud-provider", "node", "scheduling", "testing"],
-    "feature": true,
-    "action_required": true,
-    "release_version": "1.14.9"
-  },
-  "82705": {
-    "commit": "a5fe905be420d518892e8e8b682902deef82d1c6",
-    "text": "configmaps/extension-apiserver-authentication in kube-system is continuously updated by kube-apiservers, instead of just at apiserver start",
-    "markdown": "configmaps/extension-apiserver-authentication in kube-system is continuously updated by kube-apiservers, instead of just at apiserver start ([#82705](https://github.com/kubernetes/kubernetes/pull/82705), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82705",
-    "pr_number": 82705,
-    "areas": ["apiserver", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "82795": {
-    "commit": "7dc298af423f521451d6dc34b9ba803c8b46daae",
-    "text": "Graduate ScheduleDaemonSetPods to GA. (feature gate will be removed in 1.18) action required.",
-    "markdown": "Graduate ScheduleDaemonSetPods to GA. (feature gate will be removed in 1.18) action required. ([#82795](https://github.com/kubernetes/kubernetes/pull/82795), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Apps, SIG Scheduling, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82795",
-    "pr_number": 82795,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["apps", "scheduling", "testing"],
-    "feature": true,
-    "action_required": true,
-    "release_version": "1.14.9"
-  },
-  "82809": {
-    "commit": "695c3061dd92a6b6950f8adf0341ceb4a8dd44d7",
-    "text": "Kubernetes now requires go1.13.4+ to build",
-    "markdown": "Kubernetes now requires go1.13.4+ to build ([#82809](https://github.com/kubernetes/kubernetes/pull/82809), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82809",
-    "pr_number": 82809,
-    "areas": [
-      "apiserver",
-      "cloudprovider",
-      "code-generation",
-      "dependency",
-      "ipvs",
-      "kubeadm",
-      "kubectl",
-      "kubelet",
-      "release-eng",
-      "test"
-    ],
-    "kinds": ["cleanup"],
-    "sigs": [
-      "api-machinery",
-      "auth",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "network",
-      "node",
-      "release",
-      "storage",
-      "testing"
-    ],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "83195": {
-    "commit": "006378fcc1c8db4f7063812b67a59969fc0d09f0",
-    "text": "Promote WatchBookmark feature to GA.\nWith WatchBookmark feature, clients are able to request watch events with BOOKMARK type. Clients should not assume bookmarks are returned at any specific interval, nor may they assume the server will send any BOOKMARK event during a session.",
-    "markdown": "Promote WatchBookmark feature to GA.\n  With WatchBookmark feature, clients are able to request watch events with BOOKMARK type. Clients should not assume bookmarks are returned at any specific interval, nor may they assume the server will send any BOOKMARK event during a session. ([#83195](https://github.com/kubernetes/kubernetes/pull/83195), [@wojtek-t](https://github.com/wojtek-t))",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83195",
-    "pr_number": 83195,
-    "areas": ["apiserver", "code-generation"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.9"
-  },
-  "83204": {
-    "commit": "3383d7ca66cc863cc8994b9cb89f6701a7699652",
-    "text": "kubelet: a configuration file specified via `--config` is now loaded with strict deserialization, which fails if the config file contains duplicate or unknown fields. This protects against accidentally running with config files that are malformed, mis-indented, or have typos in field names, and getting unexpected behavior.",
-    "markdown": "kubelet: a configuration file specified via `--config` is now loaded with strict deserialization, which fails if the config file contains duplicate or unknown fields. This protects against accidentally running with config files that are malformed, mis-indented, or have typos in field names, and getting unexpected behavior. ([#83204](https://github.com/kubernetes/kubernetes/pull/83204), [@obitech](https://github.com/obitech))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Node",
-    "author": "obitech",
-    "author_url": "https://github.com/obitech",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83204",
-    "pr_number": 83204,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "node"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "83206": {
-    "commit": "30ba9f65480601790e85dcdbfbce5171fe180ca3",
-    "text": "Significant kube-proxy performance improvements when using Endpoint Slices at scale.",
-    "markdown": "Significant kube-proxy performance improvements when using Endpoint Slices at scale. ([#83206](https://github.com/kubernetes/kubernetes/pull/83206), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83206",
-    "pr_number": 83206,
-    "kinds": ["cleanup"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "83257": {
-    "commit": "0c32aa8910e83075071f8788828d99bcad6e7073",
-    "text": "Adding initial EndpointSlice metrics.",
-    "markdown": "Adding initial EndpointSlice metrics. ([#83257](https://github.com/kubernetes/kubernetes/pull/83257), [@robscott](https://github.com/robscott))\n\n  Courtesy of SIG Apps, and SIG Network",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83257",
-    "pr_number": 83257,
-    "kinds": ["feature"],
-    "sigs": ["apps", "network"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "83283": {
-    "commit": "7cc611b7e01191f262150def59cd39ad7eaaffed",
-    "text": "ETCD version monitor metrics are now marked as with the ALPHA stability level.",
-    "markdown": "ETCD version monitor metrics are now marked as with the ALPHA stability level. ([#83283](https://github.com/kubernetes/kubernetes/pull/83283), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/13a778c6a4a84dbde9e691e8dcf930a6eaa7ca51/keps/sig-instrumentation/20190605-metrics-stability-migration.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83283",
-    "pr_number": 83283,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "83419": {
-    "commit": "cb19b56831d54d1d31249949318ef0b07bf00df9",
-    "text": "An end-user may choose to request logs without confirming the identity of the backing kubelet.  This feature can be disabled by setting the `AllowInsecureBackendProxy` feature-gate to false.",
-    "markdown": "An end-user may choose to request logs without confirming the identity of the backing kubelet.  This feature can be disabled by setting the `AllowInsecureBackendProxy` feature-gate to false. ([#83419](https://github.com/kubernetes/kubernetes/pull/83419), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83419",
-    "pr_number": 83419,
-    "areas": ["kubelet", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["apps", "node", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "83427": {
-    "commit": "aa25739da416b68e966bef3c15e4f5281ebd1e85",
-    "text": "Adds a metric apiserver_request_error_total to kube-apiserver. This metric tallies the number of request_errors encountered by verb, group, version, resource, subresource, scope, component, and code.",
-    "markdown": "Adds a metric apiserver_request_error_total to kube-apiserver. This metric tallies the number of request_errors encountered by verb, group, version, resource, subresource, scope, component, and code. ([#83427](https://github.com/kubernetes/kubernetes/pull/83427), [@logicalhan](https://github.com/logicalhan))",
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83427",
-    "pr_number": 83427,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "83442": {
-    "commit": "a8e819746d0fbcbcafc715081177caa9b580d016",
-    "text": "Remove prometheus cluster monitoring addon from kube-up",
-    "markdown": "Remove prometheus cluster monitoring addon from kube-up ([#83442](https://github.com/kubernetes/kubernetes/pull/83442), [@serathius](https://github.com/serathius))",
-    "author": "serathius",
-    "author_url": "https://github.com/serathius",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83442",
-    "pr_number": 83442,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "83474": {
-    "commit": "1d1385af915098abd8bd23ab397ee0aeaa244f1e",
-    "text": "CSI Topology feature is GA. The CSINodeInfo feature gate is deprecated and will be removed in a future release. The storage.k8s.io/v1beta1 CSINode object is deprecated and will be removed in a future release.",
-    "markdown": "CSI Topology feature is GA. The CSINodeInfo feature gate is deprecated and will be removed in a future release. The storage.k8s.io/v1beta1 CSINode object is deprecated and will be removed in a future release. ([#83474](https://github.com/kubernetes/kubernetes/pull/83474), [@msau42](https://github.com/msau42))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG CLI, SIG Storage, and SIG Testing",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/ac738ac78ec2e425c54354119526bac916baeb7e/keps/sig-storage/20190124-csi-topology.md",
-        "type": "KEP"
-      },
-      {
-        "description": "[Usage]",
-        "url": "https://kubernetes.io/docs/concepts/storage/storage-classes/#volume-binding-mode",
-        "type": "official"
-      },
-      {
-        "description": "[Other doc]",
-        "url": "https://kubernetes-csi.github.io/docs/topology.html",
-        "type": "external"
-      }
-    ],
-    "author": "msau42",
-    "author_url": "https://github.com/msau42",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83474",
-    "pr_number": 83474,
-    "areas": ["apiserver", "kubectl", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "auth", "cli", "storage", "testing"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "83508": {
-    "commit": "a381f7cb3efa64395661c73961c07b02594acf64",
-    "text": "Scheduler now reports metrics on cache size including nodes, pods, and assumed pods",
-    "markdown": "Scheduler now reports metrics on cache size including nodes, pods, and assumed pods ([#83508](https://github.com/kubernetes/kubernetes/pull/83508), [@damemi](https://github.com/damemi))\n\n  Courtesy of SIG Instrumentation, and SIG Scheduling",
-    "author": "damemi",
-    "author_url": "https://github.com/damemi",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83508",
-    "pr_number": 83508,
-    "kinds": ["feature"],
-    "sigs": ["instrumentation", "scheduling"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "83517": {
-    "commit": "569a45fb02fc3065c7051cc7bc91fabf0e24b09c",
-    "text": "Utilize diagnostics tool to dump GKE windows test logs",
-    "markdown": "Utilize diagnostics tool to dump GKE windows test logs ([#83517](https://github.com/kubernetes/kubernetes/pull/83517), [@YangLu1031](https://github.com/YangLu1031))",
-    "author": "YangLu1031",
-    "author_url": "https://github.com/YangLu1031",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83517",
-    "pr_number": 83517,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.9"
-  },
-  "83520": {
-    "commit": "8ed2f4775a0d2b13c4be790cdfc1f34bc8b6522b",
-    "text": "When the go-client reflector relists, the ResourceVersion list option is set to the reflector's latest synced resource version to ensure the reflector does not \"go back in time\" and reprocess events older than it has already processed. If the the server responds with an HTTP 410 (Gone) status code response, the relist falls back to using resourceVersion=\"\".",
-    "markdown": "When the go-client reflector relists, the ResourceVersion list option is set to the reflector's latest synced resource version to ensure the reflector does not \"go back in time\" and reprocess events older than it has already processed. If the the server responds with an HTTP 410 (Gone) status code response, the relist falls back to using resourceVersion=\"\". ([#83520](https://github.com/kubernetes/kubernetes/pull/83520), [@jpbetz](https://github.com/jpbetz))",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83520",
-    "pr_number": 83520,
-    "areas": ["apiserver", "dependency", "kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "node"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "83535": {
-    "commit": "0c3424f53c9b6fe7e880c30e13c1d4aa206027f1",
-    "text": "Add a metric to track number of scheduler binding and prioritizing goroutines",
-    "markdown": "Add a metric to track number of scheduler binding and prioritizing goroutines ([#83535](https://github.com/kubernetes/kubernetes/pull/83535), [@wgliang](https://github.com/wgliang))",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83535",
-    "pr_number": 83535,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.9"
-  },
-  "83567": {
-    "commit": "1fcd46c250ea0cfe4c84bc5db63d83534101fb1d",
-    "text": "Fixed attachment of AWS volumes that have just been detached.",
-    "markdown": "Fixed attachment of AWS volumes that have just been detached. ([#83567](https://github.com/kubernetes/kubernetes/pull/83567), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83567",
-    "pr_number": 83567,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "83569": {
-    "commit": "1552ba6b0021440660e13316062ebbe14075b676",
-    "text": "Add latency and request count metrics for scheduler framework.",
-    "markdown": "Add latency and request count metrics for scheduler framework. ([#83569](https://github.com/kubernetes/kubernetes/pull/83569), [@liu-cong](https://github.com/liu-cong))\n\n  Courtesy of SIG Scheduling",
-    "author": "liu-cong",
-    "author_url": "https://github.com/liu-cong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83569",
-    "pr_number": 83569,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "83577": {
-    "commit": "f7091992c04e2d4f16f6b462d4a711e5b9eb0f22",
-    "text": "Add incoming pods metrics to scheduler queue.",
-    "markdown": "Add incoming pods metrics to scheduler queue. ([#83577](https://github.com/kubernetes/kubernetes/pull/83577), [@liu-cong](https://github.com/liu-cong))\n\n  Courtesy of SIG Scheduling",
-    "author": "liu-cong",
-    "author_url": "https://github.com/liu-cong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83577",
-    "pr_number": 83577,
-    "kinds": ["cleanup", "feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "83578": {
-    "commit": "2da4678a802696efa7e42a5902216c14470fce7a",
-    "text": "Scheduler Policy API has a new recommended apiVersion \"apiVersion: kubescheduler.config.k8s.io/v1\" which is consistent with the scheduler API group \"kubescheduler.config.k8s.io\". It holds the same API as the old apiVersion \"apiVersion: v1\".",
-    "markdown": "Scheduler Policy API has a new recommended apiVersion \"apiVersion: kubescheduler.config.k8s.io/v1\" which is consistent with the scheduler API group \"kubescheduler.config.k8s.io\". It holds the same API as the old apiVersion \"apiVersion: v1\". ([#83578](https://github.com/kubernetes/kubernetes/pull/83578), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83578",
-    "pr_number": 83578,
-    "areas": ["dependency", "test"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "83579": {
-    "commit": "0f1a8054f986d205a0202bf9b378c71e7ecc1710",
-    "text": "client-ca bundles for the all generic-apiserver based servers will dynamically reload from disk on content changes",
-    "markdown": "client-ca bundles for the all generic-apiserver based servers will dynamically reload from disk on content changes ([#83579](https://github.com/kubernetes/kubernetes/pull/83579), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83579",
-    "pr_number": 83579,
-    "areas": ["apiserver", "dependency", "kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth", "node", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "83591": {
-    "commit": "18cef954a1452dc0acda14b43a2d61e620fba53c",
-    "text": "Add data cache flushing during unmount device for GCE-PD driver in Windows Server.",
-    "markdown": "Add data cache flushing during unmount device for GCE-PD driver in Windows Server. ([#83591](https://github.com/kubernetes/kubernetes/pull/83591), [@jingxu97](https://github.com/jingxu97))",
-    "author": "jingxu97",
-    "author_url": "https://github.com/jingxu97",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83591",
-    "pr_number": 83591,
-    "kinds": ["bug"],
-    "sigs": ["storage", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "83592": {
-    "commit": "73b2c82b2864eeac5b81eb01e06c5726a11176cf",
-    "text": "A new kubelet command line option, --reserved-cpus, is introduced to explicitly define the the CPU list that will be reserved for system. For example, if --reserved-cpus=0,1,2,3 is specified, then cpu 0,1,2,3 will be reserved for the system.  On a system with 24 CPUs, the user may specify isolcpus=4-23 for the kernel option and use CPU 4-23 for the user containers.",
-    "markdown": "A new kubelet command line option, --reserved-cpus, is introduced to explicitly define the the CPU list that will be reserved for system. For example, if --reserved-cpus=0,1,2,3 is specified, then cpu 0,1,2,3 will be reserved for the system.  On a system with 24 CPUs, the user may specify isolcpus=4-23 for the kernel option and use CPU 4-23 for the user containers. ([#83592](https://github.com/kubernetes/kubernetes/pull/83592), [@jianzzha](https://github.com/jianzzha))\n\n  Courtesy of SIG API Machinery, SIG Cluster Lifecycle, and SIG Node",
-    "author": "jianzzha",
-    "author_url": "https://github.com/jianzzha",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83592",
-    "pr_number": 83592,
-    "areas": ["code-generation", "kubeadm", "kubelet"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "node"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "83619": {
-    "commit": "042b17ef197afb34f8e827d65f2658dcc1d63923",
-    "text": "Scheduler ComponentConfig fields are now pointers",
-    "markdown": "Scheduler ComponentConfig fields are now pointers ([#83619](https://github.com/kubernetes/kubernetes/pull/83619), [@damemi](https://github.com/damemi))",
-    "author": "damemi",
-    "author_url": "https://github.com/damemi",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83619",
-    "pr_number": 83619,
-    "areas": ["test"],
-    "kinds": ["api-change"],
-    "sigs": ["scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "83644": {
-    "commit": "bb6166419ac399d6b0007eb5657c6b94f182d9b0",
-    "text": "The userspace mode of kube-proxy no longer confusingly logs messages about deleting endpoints that it is actually adding.",
-    "markdown": "The userspace mode of kube-proxy no longer confusingly logs messages about deleting endpoints that it is actually adding. ([#83644](https://github.com/kubernetes/kubernetes/pull/83644), [@danwinship](https://github.com/danwinship))",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83644",
-    "pr_number": 83644,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.9"
-  },
-  "83650": {
-    "commit": "a78340410e519f4e9870e1b29316572e63e59a91",
-    "text": "[migration phase 1] PodFitsResources as framework plugin",
-    "markdown": "[migration phase 1] PodFitsResources as framework plugin ([#83650](https://github.com/kubernetes/kubernetes/pull/83650), [@wgliang](https://github.com/wgliang))",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83650",
-    "pr_number": 83650,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.9"
-  },
-  "83659": {
-    "commit": "5a427e87043e473e2768556de4560d51bb385e6a",
-    "text": "[migration phase 1] PodFitsHostPorts as filter plugin",
-    "markdown": "[migration phase 1] PodFitsHostPorts as filter plugin ([#83659](https://github.com/kubernetes/kubernetes/pull/83659), [@wgliang](https://github.com/wgliang))",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83659",
-    "pr_number": 83659,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.9"
-  },
-  "83660": {
-    "commit": "becc1b945b54d982fc31457ac550923895192cbd",
-    "text": "[migration phase 1] PodMatchNodeSelector/NodAffinity as filter plugin",
-    "markdown": "[migration phase 1] PodMatchNodeSelector/NodAffinity as filter plugin ([#83660](https://github.com/kubernetes/kubernetes/pull/83660), [@wgliang](https://github.com/wgliang))",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83660",
-    "pr_number": 83660,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.9"
-  },
-  "83677": {
-    "commit": "bbe71214a6800ad7e3ee03bf564e4afcfb40de77",
-    "text": "Updated kube-proxy ipvs README with correct grep argument to list loaded ipvs modules",
-    "markdown": "Updated kube-proxy ipvs README with correct grep argument to list loaded ipvs modules ([#83677](https://github.com/kubernetes/kubernetes/pull/83677), [@pete911](https://github.com/pete911))",
-    "author": "pete911",
-    "author_url": "https://github.com/pete911",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83677",
-    "pr_number": 83677,
-    "areas": ["ipvs"],
-    "kinds": ["documentation"],
-    "sigs": ["network"],
-    "release_version": "1.14.9"
-  },
-  "83685": {
-    "commit": "11ab52a74a4651ba7179123a478f9a9a8d503285",
-    "text": "azure: Add allow unsafe read from cache",
-    "markdown": "azure: Add allow unsafe read from cache ([#83685](https://github.com/kubernetes/kubernetes/pull/83685), [@aramase](https://github.com/aramase))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "aramase",
-    "author_url": "https://github.com/aramase",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83685",
-    "pr_number": 83685,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug", "feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "83692": {
-    "commit": "486e2380bd2847f1af7f3c1d0c7b01e7e688415e",
-    "text": "kube-scheduler now fallbacks to emitting events using core/v1 Events when events.k8s.io/v1beta1 is disabled.",
-    "markdown": "kube-scheduler now fallbacks to emitting events using core/v1 Events when events.k8s.io/v1beta1 is disabled. ([#83692](https://github.com/kubernetes/kubernetes/pull/83692), [@yastij](https://github.com/yastij))",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83692",
-    "pr_number": 83692,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "apps", "scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "83756": {
-    "commit": "7e53c9d80808e44ae0b320f01fe46a03eda2ccf6",
-    "text": "Refactor scheduler's framework permit API.",
-    "markdown": "Refactor scheduler's framework permit API. ([#83756](https://github.com/kubernetes/kubernetes/pull/83756), [@hex108](https://github.com/hex108))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "hex108",
-    "author_url": "https://github.com/hex108",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83756",
-    "pr_number": 83756,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "83777": {
-    "commit": "002dbf6a4c67fb73ff7a6adec88ae82f03a82cbc",
-    "text": "Fixed a bug in the single-numa-policy of the TopologyManager. Previously, best-effort pods would result in a terminated state with a TopologyAffinity error. Now they will run as expected.",
-    "markdown": "Fixed a bug in the single-numa-policy of the TopologyManager. Previously, best-effort pods would result in a terminated state with a TopologyAffinity error. Now they will run as expected. ([#83777](https://github.com/kubernetes/kubernetes/pull/83777), [@lmdaly](https://github.com/lmdaly))",
-    "author": "lmdaly",
-    "author_url": "https://github.com/lmdaly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83777",
-    "pr_number": 83777,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.14.9"
-  },
-  "83801": {
-    "commit": "5e2650883af75cfb09eb4a65b4fb0e7d867a104f",
-    "text": "Upgrade to etcd client 3.3.17 to fix bug where etcd client does not parse IPv6 addresses correctly when members are joining, and to fix bug where failover on multi-member etcd cluster fails certificate check on DNS mismatch",
-    "markdown": "Upgrade to etcd client 3.3.17 to fix bug where etcd client does not parse IPv6 addresses correctly when members are joining, and to fix bug where failover on multi-member etcd cluster fails certificate check on DNS mismatch ([#83801](https://github.com/kubernetes/kubernetes/pull/83801), [@jpbetz](https://github.com/jpbetz))",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83801",
-    "pr_number": 83801,
-    "areas": ["apiserver", "cloudprovider", "dependency", "etcd"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "83804": {
-    "commit": "cb3b715de2695044e998bce5377800a7ea6558c1",
-    "text": "Upgrade default etcd server version to 3.3.17",
-    "markdown": "Upgrade default etcd server version to 3.3.17 ([#83804](https://github.com/kubernetes/kubernetes/pull/83804), [@jpbetz](https://github.com/jpbetz))",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83804",
-    "pr_number": 83804,
-    "areas": ["etcd", "kubeadm", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "83822": {
-    "commit": "1732b435b33e0d7061f928f531ca47557045e589",
-    "text": "set config.BindAddress to IPv4 address \"127.0.0.1\" if not specified",
-    "markdown": "set config.BindAddress to IPv4 address \"127.0.0.1\" if not specified ([#83822](https://github.com/kubernetes/kubernetes/pull/83822), [@zouyee](https://github.com/zouyee))",
-    "author": "zouyee",
-    "author_url": "https://github.com/zouyee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83822",
-    "pr_number": 83822,
-    "areas": ["ipvs"],
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.14.9"
-  },
-  "83825": {
-    "commit": "2afb403fa1a0b7f3b916bf6fad35aeb1b9a22d25",
-    "text": "Gives the right error message when using `kubectl delete` a wrong resource.",
-    "markdown": "Gives the right error message when using `kubectl delete` a wrong resource. ([#83825](https://github.com/kubernetes/kubernetes/pull/83825), [@zhouya0](https://github.com/zhouya0))",
-    "author": "zhouya0",
-    "author_url": "https://github.com/zhouya0",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83825",
-    "pr_number": 83825,
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.14.9"
-  },
-  "83832": {
-    "commit": "dc8880908f1799f3a42739817c5f4ac10383facb",
-    "text": "deprecate cleanup-ipvs flag",
-    "markdown": "deprecate cleanup-ipvs flag ([#83832](https://github.com/kubernetes/kubernetes/pull/83832), [@gongguan](https://github.com/gongguan))",
-    "author": "gongguan",
-    "author_url": "https://github.com/gongguan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83832",
-    "pr_number": 83832,
-    "areas": ["ipvs"],
-    "kinds": ["cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.14.9"
-  },
-  "83902": {
-    "commit": "78abdf53750d9c76a9d6afaf928fe1387cf06a98",
-    "text": "clean duplicate GetPodServiceMemberships function",
-    "markdown": "clean duplicate GetPodServiceMemberships function ([#83902](https://github.com/kubernetes/kubernetes/pull/83902), [@gongguan](https://github.com/gongguan))",
-    "author": "gongguan",
-    "author_url": "https://github.com/gongguan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83902",
-    "pr_number": 83902,
-    "kinds": ["cleanup"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "83907": {
-    "commit": "83c6bf361d6a060181e3e5c0d1e7cf1e3dc3da7b",
-    "text": "Bumps metrics-server version to v0.3.6 with following bugfix:\n\n-  Don't break metric storage when duplicate pod metrics encountered causing hpa to fail",
-    "markdown": "Bumps metrics-server version to v0.3.6 with following bugfix:\n  \n  -  Don't break metric storage when duplicate pod metrics encountered causing hpa to fail ([#83907](https://github.com/kubernetes/kubernetes/pull/83907), [@olagacek](https://github.com/olagacek))",
-    "author": "olagacek",
-    "author_url": "https://github.com/olagacek",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83907",
-    "pr_number": 83907,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "83910": {
-    "commit": "da552df5a06cc476c31a0a8772c77fe375a4bf57",
-    "text": "Allow dynamically set glog logging level of kube-scheduler",
-    "markdown": "Allow dynamically set glog logging level of kube-scheduler ([#83910](https://github.com/kubernetes/kubernetes/pull/83910), [@mrkm4ntr](https://github.com/mrkm4ntr))\n\n  Courtesy of SIG Scheduling",
-    "author": "mrkm4ntr",
-    "author_url": "https://github.com/mrkm4ntr",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83910",
-    "pr_number": 83910,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "83911": {
-    "commit": "634bddddf271868ed8c0a9fbd853d62e0c0446bb",
-    "text": "Fixed an issue with informers missing an `Added` event if a recently deleted object was immediately recreated at the same time the informer dropped a watch and relisted.",
-    "markdown": "Fixed an issue with informers missing an `Added` event if a recently deleted object was immediately recreated at the same time the informer dropped a watch and relisted. ([#83911](https://github.com/kubernetes/kubernetes/pull/83911), [@matte21](https://github.com/matte21))",
-    "author": "matte21",
-    "author_url": "https://github.com/matte21",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83911",
-    "pr_number": 83911,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.9"
-  },
-  "83932": {
-    "commit": "b4d3dc371889452f9be22eaa4617e4da45bf2931",
-    "text": "kubeadm: enhance certs check-expiration to show the expiration info of related CAs",
-    "markdown": "kubeadm: enhance certs check-expiration to show the expiration info of related CAs ([#83932](https://github.com/kubernetes/kubernetes/pull/83932), [@SataQiu](https://github.com/SataQiu))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83932",
-    "pr_number": 83932,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "83944": {
-    "commit": "57353d524253fed5ef267101d47c230cc05a4894",
-    "text": "Added the `crictl` Windows binaries as well as the Linux 32bit binary to the release archives",
-    "markdown": "Added the `crictl` Windows binaries as well as the Linux 32bit binary to the release archives ([#83944](https://github.com/kubernetes/kubernetes/pull/83944), [@saschagrunert](https://github.com/saschagrunert))",
-    "documentation": [
-      {
-        "description": "crictl release archives",
-        "url": "https://github.com/kubernetes-sigs/cri-tools/releases/tag/v1.16.1",
-        "type": "external"
-      }
-    ],
-    "author": "saschagrunert",
-    "author_url": "https://github.com/saschagrunert",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83944",
-    "pr_number": 83944,
-    "areas": ["release-eng"],
-    "kinds": ["cleanup"],
-    "sigs": ["release"],
-    "release_version": "1.14.9"
-  },
-  "83945": {
-    "commit": "cd7a0ee8eba749551366c082c15e54aed407d101",
-    "text": "client-go: improved allocation behavior of the delaying workqueue when handling objects with far-future ready times.",
-    "markdown": "client-go: improved allocation behavior of the delaying workqueue when handling objects with far-future ready times. ([#83945](https://github.com/kubernetes/kubernetes/pull/83945), [@barkbay](https://github.com/barkbay))",
-    "author": "barkbay",
-    "author_url": "https://github.com/barkbay",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83945",
-    "pr_number": 83945,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.9"
-  },
-  "83950": {
-    "commit": "4b58ef0e14cc3eead5954b9d08313e164efba2a7",
-    "text": "kubeadm no longer removes /etc/cni/net.d as it does not install it. Users should remove files from it manually or rely on the component that created them",
-    "markdown": "kubeadm no longer removes /etc/cni/net.d as it does not install it. Users should remove files from it manually or rely on the component that created them ([#83950](https://github.com/kubernetes/kubernetes/pull/83950), [@yastij](https://github.com/yastij))",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83950",
-    "pr_number": 83950,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.9"
-  },
-  "83951": {
-    "commit": "064458de46189749da9102015312b857c47c963d",
-    "text": "Fix handling tombstones in pod-disruption-budged controller.",
-    "markdown": "Fix handling tombstones in pod-disruption-budged controller. ([#83951](https://github.com/kubernetes/kubernetes/pull/83951), [@zouyee](https://github.com/zouyee))",
-    "author": "zouyee",
-    "author_url": "https://github.com/zouyee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83951",
-    "pr_number": 83951,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.14.9"
-  },
-  "83956": {
-    "commit": "554412c62cf84a8c096399d61bb3837d00b82966",
-    "text": "Switched intstr.Type to sized integer to follow API guidelines and improve compatibility with proto libraries",
-    "markdown": "Switched intstr.Type to sized integer to follow API guidelines and improve compatibility with proto libraries ([#83956](https://github.com/kubernetes/kubernetes/pull/83956), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83956",
-    "pr_number": 83956,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.9"
-  },
-  "83963": {
-    "commit": "8d8a0684866d75aa195de20871a6164794b11ebc",
-    "text": "Scheduler policy configs can no longer be declared multiple times",
-    "markdown": "Scheduler policy configs can no longer be declared multiple times ([#83963](https://github.com/kubernetes/kubernetes/pull/83963), [@damemi](https://github.com/damemi))",
-    "author": "damemi",
-    "author_url": "https://github.com/damemi",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83963",
-    "pr_number": 83963,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.9"
-  },
-  "83987": {
-    "commit": "09f453ff8322979ed5a7611bc2e5528506c1fc7f",
-    "text": "Update etcd client side to v3.4.3\nDeprecated prometheus request meta-metrics have been removed (http_request_duration_microseconds, http_request_duration_microseconds_sum, http_request_duration_microseconds_count, http_request_size_bytes, http_request_size_bytes_sum, http_request_size_bytes_count, http_requests_total, http_response_size_bytes, http_response_size_bytes_sum, http_response_size_bytes_count) due to removal from the prometheus client library. Prometheus http request meta-metrics are now generated from [promhttp.InstrumentMetricHandler](https://godoc.org/github.com/prometheus/client_golang/prometheus/promhttp\u0026#35;InstrumentMetricHandler) instead.",
-    "markdown": "Update etcd client side to v3.4.3\n  Deprecated prometheus request meta-metrics have been removed (http_request_duration_microseconds, http_request_duration_microseconds_sum, http_request_duration_microseconds_count, http_request_size_bytes, http_request_size_bytes_sum, http_request_size_bytes_count, http_requests_total, http_response_size_bytes, http_response_size_bytes_sum, http_response_size_bytes_count) due to removal from the prometheus client library. Prometheus http request meta-metrics are now generated from [promhttp.InstrumentMetricHandler](https://godoc.org/github.com/prometheus/client_golang/prometheus/promhttp\u0026#35;InstrumentMetricHandler) instead. ([#83987](https://github.com/kubernetes/kubernetes/pull/83987), [@wenjiaswe](https://github.com/wenjiaswe))",
-    "author": "wenjiaswe",
-    "author_url": "https://github.com/wenjiaswe",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83987",
-    "pr_number": 83987,
-    "areas": [
-      "apiserver",
-      "cloudprovider",
-      "code-generation",
-      "dependency",
-      "e2e-test-framework",
-      "kubeadm",
-      "kubectl",
-      "kubelet",
-      "test"
-    ],
-    "kinds": ["api-change"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "auth",
-      "autoscaling",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "network",
-      "node",
-      "scheduling",
-      "storage",
-      "testing"
-    ],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "84007": {
-    "commit": "8ff16f35f8fd2cc504fba25be9d45f0e7dce5da3",
-    "text": "Reduce default NodeStatusReportFrequency to 5 minutes. With this change, periodic node status updates will be send every 5m if node status doesn't change (otherwise they are still send with 10s).\n\nBump NodeProblemDetector version to v0.8.0 to reduce forced NodeStatus updates frequency to 5 minutes.",
-    "markdown": "Reduce default NodeStatusReportFrequency to 5 minutes. With this change, periodic node status updates will be send every 5m if node status doesn't change (otherwise they are still send with 10s).\n  \n  Bump NodeProblemDetector version to v0.8.0 to reduce forced NodeStatus updates frequency to 5 minutes. ([#84007](https://github.com/kubernetes/kubernetes/pull/84007), [@wojtek-t](https://github.com/wojtek-t))\n\n  Courtesy of SIG Cluster Lifecycle, SIG Node, SIG Scalability, and SIG Testing",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84007",
-    "pr_number": 84007,
-    "areas": ["kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "node", "scalability", "testing"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84011": {
-    "commit": "4fdf5826877c655041b01764c3447b154bff7c33",
-    "text": "Add permit_wait_duration_seconds metric for scheduler.",
-    "markdown": "Add permit_wait_duration_seconds metric for scheduler. ([#84011](https://github.com/kubernetes/kubernetes/pull/84011), [@liu-cong](https://github.com/liu-cong))\n\n  Courtesy of SIG Scheduling",
-    "author": "liu-cong",
-    "author_url": "https://github.com/liu-cong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84011",
-    "pr_number": 84011,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84018": {
-    "commit": "ec63e099ba5b08560b20f922bd610a15bcfff2cc",
-    "text": "Update to Ingress-GCE v1.6.1",
-    "markdown": "Update to Ingress-GCE v1.6.1 ([#84018](https://github.com/kubernetes/kubernetes/pull/84018), [@rramkumar1](https://github.com/rramkumar1))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "rramkumar1",
-    "author_url": "https://github.com/rramkumar1",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84018",
-    "pr_number": 84018,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84038": {
-    "commit": "54a30700a38452a5113adcfba0f98adcc5e05f2d",
-    "text": "Update Cluster Autoscaler version to 1.16.2 (CA release docs: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.16.2)",
-    "markdown": "Update Cluster Autoscaler version to 1.16.2 (CA release docs: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.16.2) ([#84038](https://github.com/kubernetes/kubernetes/pull/84038), [@losipiuk](https://github.com/losipiuk))",
-    "author": "losipiuk",
-    "author_url": "https://github.com/losipiuk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84038",
-    "pr_number": 84038,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.9"
-  },
-  "84049": {
-    "commit": "4984c6f00033c5649710d549d5917623d2cf3cc1",
-    "text": "Fixed binding of block PersistentVolumes / PersistentVolumeClaims when BlockVolume feature is off.",
-    "markdown": "Fixed binding of block PersistentVolumes / PersistentVolumeClaims when BlockVolume feature is off. ([#84049](https://github.com/kubernetes/kubernetes/pull/84049), [@jsafrane](https://github.com/jsafrane))\n\n  Courtesy of SIG Apps, and SIG Storage",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/b1b2751/keps/sig-storage/20191008-raw-block-support.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84049",
-    "pr_number": 84049,
-    "kinds": ["feature"],
-    "sigs": ["apps", "storage"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84064": {
-    "commit": "578d0616669b53be13df0bc8805daf64ae9d01e3",
-    "text": "Update to use go1.12.12",
-    "markdown": "Update to use go1.12.12 ([#84064](https://github.com/kubernetes/kubernetes/pull/84064), [@cblecker](https://github.com/cblecker))",
-    "author": "cblecker",
-    "author_url": "https://github.com/cblecker",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84064",
-    "pr_number": 84064,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["release", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "84084": {
-    "commit": "badcd4af3f592376ce891b7c1b7a43ed6a18a348",
-    "text": "Migrate controller-manager and scheduler to EndpointsLeases leader election.",
-    "markdown": "Migrate controller-manager and scheduler to EndpointsLeases leader election. ([#84084](https://github.com/kubernetes/kubernetes/pull/84084), [@wojtek-t](https://github.com/wojtek-t))",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84084",
-    "pr_number": 84084,
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "apps", "auth", "cloud-provider", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "84091": {
-    "commit": "a7cc252b12c59d73a5e89dd03dba7c878a4a1170",
-    "text": "Adds FQDN addressType support for EndpointSlice.",
-    "markdown": "Adds FQDN addressType support for EndpointSlice. ([#84091](https://github.com/kubernetes/kubernetes/pull/84091), [@robscott](https://github.com/robscott))\n\n  Courtesy of SIG API Machinery, and SIG Network",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84091",
-    "pr_number": 84091,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "network"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84113": {
-    "commit": "e1dc959881508f64c5f8a2e6018ed49fa7c619b4",
-    "text": "Introduce x-kubernetes-map-type annotation as a CRD API extension. Enables this particular validation for server-side apply.",
-    "markdown": "Introduce x-kubernetes-map-type annotation as a CRD API extension. Enables this particular validation for server-side apply. ([#84113](https://github.com/kubernetes/kubernetes/pull/84113), [@enxebre](https://github.com/enxebre))",
-    "author": "enxebre",
-    "author_url": "https://github.com/enxebre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84113",
-    "pr_number": 84113,
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.9"
-  },
-  "84120": {
-    "commit": "1d489b77e541aecd55f641ebac888c3a3b940fd8",
-    "text": "A new `kubelet_preemptions` metric is reported from Kubelets to track the number of preemptions occuring over time, and which resource is triggering those preemptions.",
-    "markdown": "A new `kubelet_preemptions` metric is reported from Kubelets to track the number of preemptions occuring over time, and which resource is triggering those preemptions. ([#84120](https://github.com/kubernetes/kubernetes/pull/84120), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84120",
-    "pr_number": 84120,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["instrumentation", "node", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "84129": {
-    "commit": "3e5f6bdaa48723a91a208650e8d2d601fc1a55a9",
-    "text": "kube-scheduler: emits a warning when a malformed component config file is used with v1alpha1.",
-    "markdown": "kube-scheduler: emits a warning when a malformed component config file is used with v1alpha1. ([#84129](https://github.com/kubernetes/kubernetes/pull/84129), [@obitech](https://github.com/obitech))",
-    "author": "obitech",
-    "author_url": "https://github.com/obitech",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84129",
-    "pr_number": 84129,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "84138": {
-    "commit": "9790262f128aa9d27733dac46619af1d18459439",
-    "text": "None.",
-    "markdown": "None. ([#84138](https://github.com/kubernetes/kubernetes/pull/84138), [@nilo19](https://github.com/nilo19))",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84138",
-    "pr_number": 84138,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.9"
-  },
-  "84143": {
-    "commit": "7fd399e4be691c32fc413c380d67656123754780",
-    "text": "kube-proxy: emits a warning when a malformed component config file is used with v1alpha1.",
-    "markdown": "kube-proxy: emits a warning when a malformed component config file is used with v1alpha1. ([#84143](https://github.com/kubernetes/kubernetes/pull/84143), [@phenixblue](https://github.com/phenixblue))",
-    "author": "phenixblue",
-    "author_url": "https://github.com/phenixblue",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84143",
-    "pr_number": 84143,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.9"
-  },
-  "84148": {
-    "commit": "ffffd6b6f55d15c6868bc2080419740dc7a0d93d",
-    "text": "filter plugin for cloud provider storage predicate",
-    "markdown": "filter plugin for cloud provider storage predicate ([#84148](https://github.com/kubernetes/kubernetes/pull/84148), [@gongguan](https://github.com/gongguan))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "gongguan",
-    "author_url": "https://github.com/gongguan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84148",
-    "pr_number": 84148,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84152": {
-    "commit": "97c7ccc54729e7ab31775902697c21663f3882b0",
-    "text": "TaintNodesByCondition was graduated to GA, CheckNodeMemoryPressure, CheckNodePIDPressure, CheckNodeDiskPressure, CheckNodeCondition were accidentally removed since 1.12, the replacement is to use CheckNodeUnschedulablePred",
-    "markdown": "TaintNodesByCondition was graduated to GA, CheckNodeMemoryPressure, CheckNodePIDPressure, CheckNodeDiskPressure, CheckNodeCondition were accidentally removed since 1.12, the replacement is to use CheckNodeUnschedulablePred ([#84152](https://github.com/kubernetes/kubernetes/pull/84152), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84152",
-    "pr_number": 84152,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.9"
-  },
-  "84156": {
-    "commit": "6d314f860ebd0ea5b7c6462aa65347741f2747e8",
-    "text": "Fix kubelet metrics gathering on non-English Windows hosts",
-    "markdown": "Fix kubelet metrics gathering on non-English Windows hosts ([#84156](https://github.com/kubernetes/kubernetes/pull/84156), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84156",
-    "pr_number": 84156,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "windows"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "84173": {
-    "commit": "24ae4d6718f679e6331097811ae77fcbf275e124",
-    "text": "local: support local volume block mode reconstruction",
-    "markdown": "local: support local volume block mode reconstruction ([#84173](https://github.com/kubernetes/kubernetes/pull/84173), [@cofyc](https://github.com/cofyc))",
-    "author": "cofyc",
-    "author_url": "https://github.com/cofyc",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84173",
-    "pr_number": 84173,
-    "areas": ["kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["node", "storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "84181": {
-    "commit": "5f03d33fc938e792b66ce1a1062b69a075194d4d",
-    "text": "Reduced frequency of DescribeVolumes calls of AWS API when attaching/detaching a volume.",
-    "markdown": "Reduced frequency of DescribeVolumes calls of AWS API when attaching/detaching a volume. ([#84181](https://github.com/kubernetes/kubernetes/pull/84181), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84181",
-    "pr_number": 84181,
-    "areas": ["cloudprovider"],
-    "kinds": ["flake"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "84200": {
-    "commit": "04632e84e1348eb298027bae917cfede920c10a2",
-    "text": "Reload apiserver serving certificate from disk every minute",
-    "markdown": "Reload apiserver serving certificate from disk every minute ([#84200](https://github.com/kubernetes/kubernetes/pull/84200), [@jackkleeman](https://github.com/jackkleeman))\n\n  Courtesy of SIG API Machinery, SIG Auth, SIG Node, and SIG Testing",
-    "author": "jackkleeman",
-    "author_url": "https://github.com/jackkleeman",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84200",
-    "pr_number": 84200,
-    "areas": ["apiserver", "kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "auth", "node", "testing"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84207": {
-    "commit": "ed19c74f39550ddb56d5aad0702f8b5452b40efa",
-    "text": "EndpointSlice hostname is now set in the same conditions Endpoints hostname is.",
-    "markdown": "EndpointSlice hostname is now set in the same conditions Endpoints hostname is. ([#84207](https://github.com/kubernetes/kubernetes/pull/84207), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84207",
-    "pr_number": 84207,
-    "kinds": ["bug"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "84218": {
-    "commit": "ed5b03831385b0071b127ff6c329ae344b9780f6",
-    "text": "local: support local filesystem volume with block resource reconstruction",
-    "markdown": "local: support local filesystem volume with block resource reconstruction ([#84218](https://github.com/kubernetes/kubernetes/pull/84218), [@cofyc](https://github.com/cofyc))\n\n  Courtesy of SIG Node, SIG Storage, and SIG Testing",
-    "author": "cofyc",
-    "author_url": "https://github.com/cofyc",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84218",
-    "pr_number": 84218,
-    "areas": ["kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["node", "storage", "testing"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84260": {
-    "commit": "06252a4630f5c52d8abfc3b3e36c55964d106650",
-    "text": "Pod labels can no longer be updated through the pod/status updates by nodes.",
-    "markdown": "Pod labels can no longer be updated through the pod/status updates by nodes. ([#84260](https://github.com/kubernetes/kubernetes/pull/84260), [@tallclair](https://github.com/tallclair))\n\n  Courtesy of SIG Auth, and SIG Node",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-auth/20190916-noderestriction-pods.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84260",
-    "pr_number": 84260,
-    "kinds": ["feature"],
-    "sigs": ["auth", "node"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84264": {
-    "commit": "2905275a08f8e3999e7e6bc8b02f407fd6a0ff36",
-    "text": "Optimize inter-pod affinity preferredDuringSchedulingIgnoredDuringExecution type, up to 4x in some cases.",
-    "markdown": "Optimize inter-pod affinity preferredDuringSchedulingIgnoredDuringExecution type, up to 4x in some cases. ([#84264](https://github.com/kubernetes/kubernetes/pull/84264), [@ahg-g](https://github.com/ahg-g))\n\n  Courtesy of SIG Scheduling",
-    "author": "ahg-g",
-    "author_url": "https://github.com/ahg-g",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84264",
-    "pr_number": 84264,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84280": {
-    "commit": "24fb2c1afd51069a526e4c36ea5d3af993fd6b26",
-    "text": "Improving the performance of Endpoint and EndpointSlice controllers by caching Service Selectors",
-    "markdown": "Improving the performance of Endpoint and EndpointSlice controllers by caching Service Selectors ([#84280](https://github.com/kubernetes/kubernetes/pull/84280), [@gongguan](https://github.com/gongguan))",
-    "author": "gongguan",
-    "author_url": "https://github.com/gongguan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84280",
-    "pr_number": 84280,
-    "kinds": ["bug"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "84282": {
-    "commit": "a3560d3ad9a7e2deb7d8b7e9e54081e7cbbac0d1",
-    "text": "The built-in system:csi-external-provisioner and system:csi-external-attacher cluster roles are removed as of 1.17 release",
-    "markdown": "The built-in system:csi-external-provisioner and system:csi-external-attacher cluster roles are removed as of 1.17 release ([#84282](https://github.com/kubernetes/kubernetes/pull/84282), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84282",
-    "pr_number": 84282,
-    "kinds": ["cleanup"],
-    "sigs": ["auth", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "84297": {
-    "commit": "41730db3f0637a9175a41a435b2a6e7051f73f2b",
-    "text": "User can now use component config to configure NodeLabel plugin for the scheduler framework.",
-    "markdown": "User can now use component config to configure NodeLabel plugin for the scheduler framework. ([#84297](https://github.com/kubernetes/kubernetes/pull/84297), [@liu-cong](https://github.com/liu-cong))\n\n  Courtesy of SIG Scheduling",
-    "author": "liu-cong",
-    "author_url": "https://github.com/liu-cong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84297",
-    "pr_number": 84297,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84303": {
-    "commit": "c7bb076ade3e91cc13a0e855e155484fb4040191",
-    "text": "Reload apiserver SNI certificates from disk every minute",
-    "markdown": "Reload apiserver SNI certificates from disk every minute ([#84303](https://github.com/kubernetes/kubernetes/pull/84303), [@jackkleeman](https://github.com/jackkleeman))\n\n  Courtesy of SIG API Machinery, and SIG Testing",
-    "author": "jackkleeman",
-    "author_url": "https://github.com/jackkleeman",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84303",
-    "pr_number": 84303,
-    "areas": ["apiserver", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "testing"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84321": {
-    "commit": "cb0f80efe7982741e19dad9f1a9fa91797f57ed3",
-    "text": "CSI detach timeout increased from 10 seconds to 2 minutes",
-    "markdown": "CSI detach timeout increased from 10 seconds to 2 minutes ([#84321](https://github.com/kubernetes/kubernetes/pull/84321), [@cduchesne](https://github.com/cduchesne))",
-    "author": "cduchesne",
-    "author_url": "https://github.com/cduchesne",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84321",
-    "pr_number": 84321,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.14.9"
-  },
-  "84329": {
-    "commit": "e1d3cc74e554f37765e81fdf265cd1318dc451ae",
-    "text": "Update default etcd server version to 3.4.3",
-    "markdown": "Update default etcd server version to 3.4.3 ([#84329](https://github.com/kubernetes/kubernetes/pull/84329), [@jingyih](https://github.com/jingyih))",
-    "author": "jingyih",
-    "author_url": "https://github.com/jingyih",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84329",
-    "pr_number": 84329,
-    "areas": ["e2e-test-framework", "etcd", "kubeadm", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "84351": {
-    "commit": "ae1536835545c8d9fdb896df10cc3439c3242445",
-    "text": "Promote NodeLease feature to GA.\nThe feature make Lease object changes an additional healthiness signal from Node. Together with that, we reduce frequency of NodeStatus updates to 5m by default in case of no changes to status itself",
-    "markdown": "Promote NodeLease feature to GA.\n  The feature make Lease object changes an additional healthiness signal from Node. Together with that, we reduce frequency of NodeStatus updates to 5m by default in case of no changes to status itself ([#84351](https://github.com/kubernetes/kubernetes/pull/84351), [@wojtek-t](https://github.com/wojtek-t))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG Node, and SIG Testing",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84351",
-    "pr_number": 84351,
-    "areas": ["kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "apps", "auth", "node", "testing"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84365": {
-    "commit": "08410cbf06f1c8649157407c660b1c5cb528947b",
-    "text": "Removed Alpha feature `MountContainers`",
-    "markdown": "Removed Alpha feature `MountContainers` ([#84365](https://github.com/kubernetes/kubernetes/pull/84365), [@codenrhoden](https://github.com/codenrhoden))",
-    "author": "codenrhoden",
-    "author_url": "https://github.com/codenrhoden",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84365",
-    "pr_number": 84365,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node", "storage"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "84383": {
-    "commit": "6170296b66b3660f2f2f62644c3cd0d8c215f3c1",
-    "text": "This PR sets the --cluster-dns flag value to kube-dns service IP whether or not NodeLocal DNSCache is enabled. NodeLocal DNSCache will listen on both the link-local as well as the service IP.",
-    "markdown": "This PR sets the --cluster-dns flag value to kube-dns service IP whether or not NodeLocal DNSCache is enabled. NodeLocal DNSCache will listen on both the link-local as well as the service IP. ([#84383](https://github.com/kubernetes/kubernetes/pull/84383), [@prameshj](https://github.com/prameshj))",
-    "author": "prameshj",
-    "author_url": "https://github.com/prameshj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84383",
-    "pr_number": 84383,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "84421": {
-    "commit": "4002e4c7e7b12b052310bce808117d2f2472ad4b",
-    "text": "Fix the bug that EndpointSlice for masters wasn't created after enabling EndpointSlice feature on a pre-existing cluster.",
-    "markdown": "Fix the bug that EndpointSlice for masters wasn't created after enabling EndpointSlice feature on a pre-existing cluster. ([#84421](https://github.com/kubernetes/kubernetes/pull/84421), [@tnqn](https://github.com/tnqn))",
-    "author": "tnqn",
-    "author_url": "https://github.com/tnqn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84421",
-    "pr_number": 84421,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.9"
-  },
-  "84468": {
-    "commit": "b6c8f4916dc3a625e9decc2fd273e2a522e8b474",
-    "text": "kubeadm: always mount the kube-controller-manager hostPath volume that is given by the --flex-volume-plugin-dir flag.",
-    "markdown": "kubeadm: always mount the kube-controller-manager hostPath volume that is given by the --flex-volume-plugin-dir flag. ([#84468](https://github.com/kubernetes/kubernetes/pull/84468), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84468",
-    "pr_number": 84468,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.9"
-  },
-  "84476": {
-    "commit": "28887de8cd1b51ba2416a2e60b0b023728c5a54e",
-    "text": "update the latest validated version of Docker to 19.03",
-    "markdown": "update the latest validated version of Docker to 19.03 ([#84476](https://github.com/kubernetes/kubernetes/pull/84476), [@neolit123](https://github.com/neolit123))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84476",
-    "pr_number": 84476,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84481": {
-    "commit": "d0f2c7c5902dbeb934e52fc925b3cc0488440ee2",
-    "text": "Fixed EndpointSlice port name validation to match Endpoint port name validation (allowing port names longer than 15 characters)",
-    "markdown": "Fixed EndpointSlice port name validation to match Endpoint port name validation (allowing port names longer than 15 characters) ([#84481](https://github.com/kubernetes/kubernetes/pull/84481), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84481",
-    "pr_number": 84481,
-    "kinds": ["api-change", "bug"],
-    "sigs": ["network"],
-    "release_version": "1.14.9"
-  },
-  "84534": {
-    "commit": "0ed66351739608660fef27de55d8e0dff6570f28",
-    "text": "Added kubelet serving certificate metric `server_rotation_seconds` which is a histogram reporting the age of a just rotated serving certificate in seconds.",
-    "markdown": "Added kubelet serving certificate metric `server_rotation_seconds` which is a histogram reporting the age of a just rotated serving certificate in seconds. ([#84534](https://github.com/kubernetes/kubernetes/pull/84534), [@sambdavidson](https://github.com/sambdavidson))\n\n  Courtesy of SIG API Machinery, SIG Auth, SIG Instrumentation, and SIG Node",
-    "author": "sambdavidson",
-    "author_url": "https://github.com/sambdavidson",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84534",
-    "pr_number": 84534,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "auth", "instrumentation", "node"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84543": {
-    "commit": "4d55d1d695225f1e4953feb999616451081b2f6f",
-    "text": "Update Azure SDK versions to v35.0.0",
-    "markdown": "Update Azure SDK versions to v35.0.0 ([#84543](https://github.com/kubernetes/kubernetes/pull/84543), [@andyzhangx](https://github.com/andyzhangx))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84543",
-    "pr_number": 84543,
-    "areas": ["cloudprovider", "dependency", "provider/azure"],
-    "kinds": ["bug", "cleanup", "feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84559": {
-    "commit": "ef479c1a6fdedca9fa396dbcc19f89a983ac47a9",
-    "text": "kubeadm now propagates proxy environment variables to kube-proxy",
-    "markdown": "kubeadm now propagates proxy environment variables to kube-proxy ([#84559](https://github.com/kubernetes/kubernetes/pull/84559), [@yastij](https://github.com/yastij))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84559",
-    "pr_number": 84559,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84573": {
-    "commit": "ca836256c4234fd82b463ab03a857976141c6692",
-    "text": "Updated EndpointSlices to use PublishNotReadyAddresses from Services.",
-    "markdown": "Updated EndpointSlices to use PublishNotReadyAddresses from Services. ([#84573](https://github.com/kubernetes/kubernetes/pull/84573), [@robscott](https://github.com/robscott))\n\n  Courtesy of SIG Apps, and SIG Network",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84573",
-    "pr_number": 84573,
-    "kinds": ["feature"],
-    "sigs": ["apps", "network"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84591": {
-    "commit": "3c7706bf9c9dc1f0c2350b1f8c6d94e2536ec55e",
-    "text": "kubeadm: the command \"kubeadm token create\" now has a \"--certificate-key\" flag that can be used for the formation of join commands for control-planes with automatic copy of certificates",
-    "markdown": "kubeadm: the command \"kubeadm token create\" now has a \"--certificate-key\" flag that can be used for the formation of join commands for control-planes with automatic copy of certificates ([#84591](https://github.com/kubernetes/kubernetes/pull/84591), [@TheLastProject](https://github.com/TheLastProject))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "TheLastProject",
-    "author_url": "https://github.com/TheLastProject",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84591",
-    "pr_number": 84591,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84605": {
-    "commit": "1488460cd2c4948da5fa613655b73e92ff788c96",
-    "text": "add azure disk encryption(SSE+CMK) support",
-    "markdown": "add azure disk encryption(SSE+CMK) support ([#84605](https://github.com/kubernetes/kubernetes/pull/84605), [@andyzhangx](https://github.com/andyzhangx))\n\n  Courtesy of SIG Cloud Provider, and SIG Storage",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84605",
-    "pr_number": 84605,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug", "feature"],
-    "sigs": ["cloud-provider", "storage"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84622": {
-    "commit": "303de85dd188d16db808baba5857b7f0e6ecc1db",
-    "text": "Change GCP ILB firewall names to contain the \"k8s-fw-\" prefix like the rest of the firewall rules. This is needed for consistency and also for other components to identify the firewall rule as k8s/service-controller managed.",
-    "markdown": "Change GCP ILB firewall names to contain the \"k8s-fw-\" prefix like the rest of the firewall rules. This is needed for consistency and also for other components to identify the firewall rule as k8s/service-controller managed. ([#84622](https://github.com/kubernetes/kubernetes/pull/84622), [@prameshj](https://github.com/prameshj))",
-    "author": "prameshj",
-    "author_url": "https://github.com/prameshj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84622",
-    "pr_number": 84622,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.9"
-  },
-  "84637": {
-    "commit": "b55440dcf853aa517f991992bcb77eb1a708d59d",
-    "text": "People can see the right log and note.",
-    "markdown": "People can see the right log and note. ([#84637](https://github.com/kubernetes/kubernetes/pull/84637), [@zhipengzuo](https://github.com/zhipengzuo))",
-    "author": "zhipengzuo",
-    "author_url": "https://github.com/zhipengzuo",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84637",
-    "pr_number": 84637,
-    "kinds": ["cleanup"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "84646": {
-    "commit": "077f825c584903eeb5854040192c2977079343e9",
-    "text": "Only validate duplication of the RequestedToCapacityRatio custom priority and allow other custom predicates/priorities",
-    "markdown": "Only validate duplication of the RequestedToCapacityRatio custom priority and allow other custom predicates/priorities ([#84646](https://github.com/kubernetes/kubernetes/pull/84646), [@liu-cong](https://github.com/liu-cong))",
-    "author": "liu-cong",
-    "author_url": "https://github.com/liu-cong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84646",
-    "pr_number": 84646,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.14.9"
-  },
-  "84662": {
-    "commit": "84fcf126c1624ea95174af0492f99339fc1a8850",
-    "text": "Simple script based hyperkube image that bundles all the necessary binaries. This is a equivalent replacement for the image based on the go based hyperkube command + image.",
-    "markdown": "Simple script based hyperkube image that bundles all the necessary binaries. This is a equivalent replacement for the image based on the go based hyperkube command + image. ([#84662](https://github.com/kubernetes/kubernetes/pull/84662), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84662",
-    "pr_number": 84662,
-    "areas": ["hyperkube", "release-eng"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "release"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "84677": {
-    "commit": "aaa57078969bbb72e9ebe45198edb0899bb1f7df",
-    "text": "The certificate signer no longer accepts ca.key passwords via the CFSSL_CA_PK_PASSWORD environment variable. This capability was not prompted by user request, never advertised, and recommended against in the security audit.",
-    "markdown": "The certificate signer no longer accepts ca.key passwords via the CFSSL_CA_PK_PASSWORD environment variable. This capability was not prompted by user request, never advertised, and recommended against in the security audit. ([#84677](https://github.com/kubernetes/kubernetes/pull/84677), [@mikedanese](https://github.com/mikedanese))",
-    "author": "mikedanese",
-    "author_url": "https://github.com/mikedanese",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84677",
-    "pr_number": 84677,
-    "areas": ["dependency", "kubectl", "kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "apps", "auth", "cli", "node"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "84693": {
-    "commit": "7c7ae977d5ccc011dc2e1fcb7f06c3da2da09991",
-    "text": "kube-apiserver: fixed a bug that could cause a goroutine leak if the apiserver encountered an encoding error serving a watch to a websocket watcher",
-    "markdown": "kube-apiserver: fixed a bug that could cause a goroutine leak if the apiserver encountered an encoding error serving a watch to a websocket watcher ([#84693](https://github.com/kubernetes/kubernetes/pull/84693), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84693",
-    "pr_number": 84693,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.14.9"
-  },
-  "84713": {
-    "commit": "64be5ae5f178018d510d930457509f6b67d443c2",
-    "text": "CRDs defaulting is promoted to GA. Note: the feature gate CustomResourceDefaulting will be removed in 1.18.",
-    "markdown": "CRDs defaulting is promoted to GA. Note: the feature gate CustomResourceDefaulting will be removed in 1.18. ([#84713](https://github.com/kubernetes/kubernetes/pull/84713), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84713",
-    "pr_number": 84713,
-    "areas": ["custom-resources", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "84727": {
-    "commit": "49a9b6cadfe6f27045e9ef05179b0af1cb6e9693",
-    "text": "If given an IPv6 bind-address, kube-apiserver will now advertise an IPv6 endpoint for the kubernetes.default service.",
-    "markdown": "If given an IPv6 bind-address, kube-apiserver will now advertise an IPv6 endpoint for the kubernetes.default service. ([#84727](https://github.com/kubernetes/kubernetes/pull/84727), [@danwinship](https://github.com/danwinship))",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84727",
-    "pr_number": 84727,
-    "areas": ["apiserver", "ipv6", "kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "network"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  },
-  "84758": {
-    "commit": "7c2bfc0dda7c46a0a6e01e96bd6fa0e99813ed8f",
-    "text": "All resources within the rbac.authorization.k8s.io/v1alpha1 and rbac.authorization.k8s.io/v1beta1 API groups are deprecated in favor of rbac.authorization.k8s.io/v1, and will no longer be served in v1.20.",
-    "markdown": "All resources within the rbac.authorization.k8s.io/v1alpha1 and rbac.authorization.k8s.io/v1beta1 API groups are deprecated in favor of rbac.authorization.k8s.io/v1, and will no longer be served in v1.20. ([#84758](https://github.com/kubernetes/kubernetes/pull/84758), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84758",
-    "pr_number": 84758,
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["auth"],
-    "release_version": "1.14.9"
-  },
-  "84802": {
-    "commit": "e450a3d6f34f03b7d93d4d6378a9cbbd0864585c",
-    "text": "Ensure health probes are created for local traffic policy UDP services on Azure",
-    "markdown": "Ensure health probes are created for local traffic policy UDP services on Azure ([#84802](https://github.com/kubernetes/kubernetes/pull/84802), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84802",
-    "pr_number": 84802,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.9"
-  },
-  "84809": {
-    "commit": "41d9522a54b1faaadf1b1814777b626a98bf7eb4",
-    "text": "CSI Migration: GCE PD access mode now reflects read only status of inline volumes - this allows multi-attach for read only many PDs",
-    "markdown": "CSI Migration: GCE PD access mode now reflects read only status of inline volumes - this allows multi-attach for read only many PDs ([#84809](https://github.com/kubernetes/kubernetes/pull/84809), [@davidz627](https://github.com/davidz627))",
-    "author": "davidz627",
-    "author_url": "https://github.com/davidz627",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84809",
-    "pr_number": 84809,
-    "kinds": ["bug", "failing-test"],
-    "sigs": ["storage"],
-    "release_version": "1.14.9"
-  },
-  "84835": {
-    "commit": "f23dd405f18f23dda4d00820ab20ec49852e5727",
-    "text": "Profiling is enabled by default in the scheduler",
-    "markdown": "Profiling is enabled by default in the scheduler ([#84835](https://github.com/kubernetes/kubernetes/pull/84835), [@denkensk](https://github.com/denkensk))\n\n  Courtesy of SIG Scheduling",
-    "author": "denkensk",
-    "author_url": "https://github.com/denkensk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84835",
-    "pr_number": 84835,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84888": {
-    "commit": "c843d9614c93cf0867e72fba01cbd532d6579361",
-    "text": "kubeadm: fix an issue with the kube-proxy container env. variables",
-    "markdown": "kubeadm: fix an issue with the kube-proxy container env. variables ([#84888](https://github.com/kubernetes/kubernetes/pull/84888), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84888",
-    "pr_number": 84888,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.14.9"
-  },
-  "84896": {
-    "commit": "d366d2eaa36e0d27fca529ea0be6648f56513fda",
-    "text": "Following metrics from kube-controller-manager are now marked as with the ALPHA stability level:\nstorage_count_attachable_volumes_in_use\nattachdetach_controller_total_volumes\npv_collector_bound_pv_count\npv_collector_unbound_pv_count\npv_collector_bound_pvc_count\npv_collector_unbound_pvc_count",
-    "markdown": "Following metrics from kube-controller-manager are now marked as with the ALPHA stability level:\n  storage_count_attachable_volumes_in_use\n  attachdetach_controller_total_volumes\n  pv_collector_bound_pv_count\n  pv_collector_unbound_pv_count\n  pv_collector_bound_pvc_count\n  pv_collector_unbound_pvc_count ([#84896](https://github.com/kubernetes/kubernetes/pull/84896), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Apps",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84896",
-    "pr_number": 84896,
-    "kinds": ["feature"],
-    "sigs": ["apps"],
-    "feature": true,
-    "release_version": "1.14.9"
-  },
-  "84917": {
-    "commit": "91a53b6869811f6b0338464779eb634da4ef1fcc",
-    "text": "fix race condition when attach/delete azure disk in same time",
-    "markdown": "fix race condition when attach/delete azure disk in same time ([#84917](https://github.com/kubernetes/kubernetes/pull/84917), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84917",
-    "pr_number": 84917,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.14.9"
-  },
-  "84963": {
-    "commit": "15f586a6c18d7ccbc01f7a1e908dbabcfd146ae2",
-    "text": "kube-apiserver: Fixed a regression accepting patch requests \u003e 1MB",
-    "markdown": "kube-apiserver: Fixed a regression accepting patch requests \u003e 1MB ([#84963](https://github.com/kubernetes/kubernetes/pull/84963), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84963",
-    "pr_number": 84963,
-    "areas": ["apiserver", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.14.9"
-  }
-}
diff --git a/src/assets/release-notes-1.15.0.json b/src/assets/release-notes-1.15.0.json
deleted file mode 100644
index 9b39b70b..00000000
--- a/src/assets/release-notes-1.15.0.json
+++ /dev/null
@@ -1,3883 +0,0 @@
-{
-  "60771": {
-    "commit": "d1e7bb297c02bbbcb46dd7eeebae41cd042bb48c",
-    "text": "Fixes a bug where the kube-apiserver won't start if a broken webhook is installed and the client ca configmap is empty",
-    "markdown": "Fixes a bug where the kube-apiserver won't start if a broken webhook is installed and the client ca configmap is empty ([#60771](https://github.com/kubernetes/kubernetes/pull/60771), [@jennybuckley](https://github.com/jennybuckley))",
-    "author": "jennybuckley",
-    "author_url": "https://github.com/jennybuckley",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/60771",
-    "pr_number": 60771,
-    "areas": ["apiserver", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "62159": {
-    "commit": "ec89794c843751899547a685f6579bca86e52e59",
-    "text": "fix bug for regular empty file",
-    "markdown": "fix bug for regular empty file ([#62159](https://github.com/kubernetes/kubernetes/pull/62159), [@dixudx](https://github.com/dixudx))",
-    "author": "dixudx",
-    "author_url": "https://github.com/dixudx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/62159",
-    "pr_number": 62159,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.15.0"
-  },
-  "65782": {
-    "commit": "e67c266a72bf7b379a2b283a9989804f2282bac5",
-    "text": "Implementing deduplication logic for v1beta1.Event API",
-    "markdown": "Implementing deduplication logic for v1beta1.Event API ([#65782](https://github.com/kubernetes/kubernetes/pull/65782), [@yastij](https://github.com/yastij))",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/65782",
-    "pr_number": 65782,
-    "areas": ["apiserver", "dependency", "release-eng"],
-    "sigs": ["api-machinery", "release"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "66635": {
-    "commit": "4e397d971a72f5bc6dbffbacca7297f17e17c572",
-    "text": "The `system:aws-cloud-provider` cluster role, deprecated in v1.13, is no longer auto-created. Deployments using the AWS cloud provider should grant required permissions to the `aws-cloud-provider` service account in the `kube-system` namespace as part of deployment. ",
-    "markdown": "The `system:aws-cloud-provider` cluster role, deprecated in v1.13, is no longer auto-created. Deployments using the AWS cloud provider should grant required permissions to the `aws-cloud-provider` service account in the `kube-system` namespace as part of deployment.  ([#66635](https://github.com/kubernetes/kubernetes/pull/66635), [@wgliang](https://github.com/wgliang))",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/66635",
-    "pr_number": 66635,
-    "sigs": ["auth", "aws", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "66928": {
-    "commit": "cf76868b3430be015cc1e8443a216bf863a9b9f7",
-    "text": "kubelet now allows use of XFS quotas (on XFS and suitably configured ext4fs filesystems) to monitor storage consumption for ephemeral storage (currently for emptydir volumes only).  This method of monitoring consumption is faster and more accurate than the old method of walking the filesystem tree.  It does not enforce limits, only monitors consumption.  To utilize this functionality, you must set the feature  gate `LocalStorageCapacityIsolationFSQuotaMonitoring=true`. For ext4fs filesystems, you must create the\nfilesystem with `mkfs.ext4 -O project \u003cblock_device\u003e` and run `tune2fs -Q prjquota `block device`; XFS\nfilesystems need no additional preparation.  The filesystem must be mounted with option `project` in\n`/etc/fstab`.  If your primary partition is the root filesystem, you must also add `rootflags=pquota` to your\nGRUB config file.",
-    "markdown": "kubelet now allows use of XFS quotas (on XFS and suitably configured ext4fs filesystems) to monitor storage consumption for ephemeral storage (currently for emptydir volumes only).  This method of monitoring consumption is faster and more accurate than the old method of walking the filesystem tree.  It does not enforce limits, only monitors consumption.  To utilize this functionality, you must set the feature  gate `LocalStorageCapacityIsolationFSQuotaMonitoring=true`. For ext4fs filesystems, you must create the\nfilesystem with `mkfs.ext4 -O project \u003cblock_device\u003e` and run `tune2fs -Q prjquota `block device`; XFS\nfilesystems need no additional preparation.  The filesystem must be mounted with option `project` in\n`/etc/fstab`.  If your primary partition is the root filesystem, you must also add `rootflags=pquota` to your\nGRUB config file. ([#66928](https://github.com/kubernetes/kubernetes/pull/66928), [@RobertKrawitz](https://github.com/RobertKrawitz)) Courtesy of SIG API Machinery, SIG Apps, SIG CLI, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Node, SIG Release, SIG Scheduling, SIG Storage, and SIG Testing",
-    "author": "RobertKrawitz",
-    "author_url": "https://github.com/RobertKrawitz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/66928",
-    "pr_number": 66928,
-    "areas": ["apiserver", "kubeadm", "kubectl", "kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "node",
-      "release",
-      "scheduling",
-      "storage",
-      "testing"
-    ],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "68108": {
-    "commit": "3d4124f2e083591c98e8a874760011781d3ee15c",
-    "text": "Minor fix ConstructVolumeSpec in iSCSI plugin",
-    "markdown": "Minor fix ConstructVolumeSpec in iSCSI plugin ([#68108](https://github.com/kubernetes/kubernetes/pull/68108), [@wenjun93](https://github.com/wenjun93))",
-    "author": "wenjun93",
-    "author_url": "https://github.com/wenjun93",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/68108",
-    "pr_number": 68108,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.15.0"
-  },
-  "69867": {
-    "commit": "de6ce5ee16895ce8aef0f0f6faa194541b7596c2",
-    "text": "Allow updates/patches to pod disruption budgets",
-    "markdown": "Allow updates/patches to pod disruption budgets ([#69867](https://github.com/kubernetes/kubernetes/pull/69867), [@davidmccormick](https://github.com/davidmccormick)) Courtesy of SIG Apps, SIG Auth, and SIG Testing",
-    "author": "davidmccormick",
-    "author_url": "https://github.com/davidmccormick",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/69867",
-    "pr_number": 69867,
-    "areas": ["apiserver", "test"],
-    "kinds": ["feature"],
-    "sigs": ["apps", "auth", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "70715": {
-    "commit": "3c3c1b11cb4afdfecc73942299243b64b5116acf",
-    "text": "The transformer_failures_total metric is deprecated in favor of transformation_operation_total. The old metric will continue to be populated but will be removed in a future release.",
-    "markdown": "The transformer_failures_total metric is deprecated in favor of transformation_operation_total. The old metric will continue to be populated but will be removed in a future release. ([#70715](https://github.com/kubernetes/kubernetes/pull/70715), [@immutableT](https://github.com/immutableT)) Courtesy of SIG API Machinery, and SIG Instrumentation",
-    "author": "immutableT",
-    "author_url": "https://github.com/immutableT",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/70715",
-    "pr_number": 70715,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "instrumentation"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "71548": {
-    "commit": "6f9bf5fe98bcc3b436fea4d6dd345a1502d20778",
-    "text": "Watch will now support converting response objects into Table or PartialObjectMetadata forms.",
-    "markdown": "Watch will now support converting response objects into Table or PartialObjectMetadata forms. ([#71548](https://github.com/kubernetes/kubernetes/pull/71548), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/71548",
-    "pr_number": 71548,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "71735": {
-    "commit": "2490e035d79aa01665597aae972fb6672ef74756",
-    "text": "The userspace proxy now respects the IPTables proxy's minSyncInterval parameter.",
-    "markdown": "The userspace proxy now respects the IPTables proxy's minSyncInterval parameter. ([#71735](https://github.com/kubernetes/kubernetes/pull/71735), [@dcbw](https://github.com/dcbw))",
-    "author": "dcbw",
-    "author_url": "https://github.com/dcbw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/71735",
-    "pr_number": 71735,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.15.0"
-  },
-  "72179": {
-    "commit": "11d472ea01f23063158e1f0e4fba37144e4edc59",
-    "text": "Transition service account controller clients to TokenRequest API",
-    "markdown": "Transition service account controller clients to TokenRequest API ([#72179](https://github.com/kubernetes/kubernetes/pull/72179), [@WanLinghao](https://github.com/WanLinghao)) Courtesy of SIG API Machinery, SIG Apps, SIG Auth, and SIG Testing",
-    "author": "WanLinghao",
-    "author_url": "https://github.com/WanLinghao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72179",
-    "pr_number": 72179,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "apps", "auth", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "72384": {
-    "commit": "b9dbb458fb919154620fc2525f56dc515b44e68e",
-    "text": "Object count quota is now supported for namespaced custom resources using the `count/\u003cresource\u003e.\u003cgroup\u003e` syntax.",
-    "markdown": "Object count quota is now supported for namespaced custom resources using the `count/\u003cresource\u003e.\u003cgroup\u003e` syntax. ([#72384](https://github.com/kubernetes/kubernetes/pull/72384), [@zhouhaibing089](https://github.com/zhouhaibing089)) Courtesy of SIG API Machinery, SIG Apps, and SIG Testing",
-    "author": "zhouhaibing089",
-    "author_url": "https://github.com/zhouhaibing089",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72384",
-    "pr_number": 72384,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "apps", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "72454": {
-    "commit": "5a6c66a729b4fa23d8fd254dbb0098396a192227",
-    "text": "In case kubeadm can't access the current Kubernetes version remotely and fails to parse\nthe git-based version it falls back to a static predefined value of\nk8s.io/kubernetes/cmd/kubeadm/app/constants.CurrentKubernetesVersion.",
-    "markdown": "In case kubeadm can't access the current Kubernetes version remotely and fails to parse\nthe git-based version it falls back to a static predefined value of\nk8s.io/kubernetes/cmd/kubeadm/app/constants.CurrentKubernetesVersion. ([#72454](https://github.com/kubernetes/kubernetes/pull/72454), [@rojkov](https://github.com/rojkov))",
-    "author": "rojkov",
-    "author_url": "https://github.com/rojkov",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72454",
-    "pr_number": 72454,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup", "failing-test"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "72787": {
-    "commit": "f49fe2a7504c40684573f3e4928e8b0bfdfb2b30",
-    "text": "The kubelet only collects metrics for the node, container runtime, kubelet, pods, and containers.",
-    "markdown": "The kubelet only collects metrics for the node, container runtime, kubelet, pods, and containers. ([#72787](https://github.com/kubernetes/kubernetes/pull/72787), [@dashpole](https://github.com/dashpole)) Courtesy of SIG Node, and SIG Testing",
-    "author": "dashpole",
-    "author_url": "https://github.com/dashpole",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72787",
-    "pr_number": 72787,
-    "areas": ["kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["node", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "73101": {
-    "commit": "aa84b99fa0a49d0206984f4deb5ad3b58817f838",
-    "text": "util/initsystem: add support for the OpenRC init system",
-    "markdown": "util/initsystem: add support for the OpenRC init system ([#73101](https://github.com/kubernetes/kubernetes/pull/73101), [@oz123](https://github.com/oz123)) Courtesy of SIG Cluster Lifecycle, and SIG Node",
-    "author": "oz123",
-    "author_url": "https://github.com/oz123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73101",
-    "pr_number": 73101,
-    "areas": ["kubeadm", "kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "node"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "73241": {
-    "commit": "e476a60ccbe25581f5a6a9401081dcee311a066e",
-    "text": "Set selinux label at plugin socket directory",
-    "markdown": "Set selinux label at plugin socket directory ([#73241](https://github.com/kubernetes/kubernetes/pull/73241), [@vikaschoudhary16](https://github.com/vikaschoudhary16))",
-    "author": "vikaschoudhary16",
-    "author_url": "https://github.com/vikaschoudhary16",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73241",
-    "pr_number": 73241,
-    "areas": ["hw-accelerators", "kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.15.0"
-  },
-  "73447": {
-    "commit": "9577cbfb9fd607d3698e1f2440f5875c85eca384",
-    "text": "Improved README for k8s.io/sample-apiserver",
-    "markdown": "Improved README for k8s.io/sample-apiserver ([#73447](https://github.com/kubernetes/kubernetes/pull/73447), [@MikeSpreitzer](https://github.com/MikeSpreitzer))",
-    "author": "MikeSpreitzer",
-    "author_url": "https://github.com/MikeSpreitzer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73447",
-    "pr_number": 73447,
-    "kinds": ["documentation"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "73664": {
-    "commit": "e5dd4521a8c938604287684109b22129b5395a3e",
-    "text": "kubectl exec now allows using resource name (e.g., deployment/mydeployment) to select a matching pod.\nkubectl exec now allows using --pod-running-timeout flag to wait till at least one pod is running.",
-    "markdown": "kubectl exec now allows using resource name (e.g., deployment/mydeployment) to select a matching pod.\nkubectl exec now allows using --pod-running-timeout flag to wait till at least one pod is running. ([#73664](https://github.com/kubernetes/kubernetes/pull/73664), [@prksu](https://github.com/prksu)) Courtesy of SIG CLI, and SIG Testing",
-    "author": "prksu",
-    "author_url": "https://github.com/prksu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73664",
-    "pr_number": 73664,
-    "areas": ["kubectl", "test"],
-    "kinds": ["feature"],
-    "sigs": ["cli", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "73795": {
-    "commit": "6a48257627d3bce70457342cb7514f36b5240dc5",
-    "text": "Add RuntimeClass restrictions \u0026 defaulting to PodSecurityPolicy.",
-    "markdown": "Add RuntimeClass restrictions \u0026 defaulting to PodSecurityPolicy. ([#73795](https://github.com/kubernetes/kubernetes/pull/73795), [@tallclair](https://github.com/tallclair)) Courtesy of SIG Apps, SIG Architecture, SIG Auth, and SIG Node",
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73795",
-    "pr_number": 73795,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["apps", "architecture", "auth", "node"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "73863": {
-    "commit": "76b1d8351720c5a889536e5db9de972d40ed43a2",
-    "text": "Count PVCs that are unbound towards attach limit",
-    "markdown": "Count PVCs that are unbound towards attach limit ([#73863](https://github.com/kubernetes/kubernetes/pull/73863), [@gnufied](https://github.com/gnufied))",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73863",
-    "pr_number": 73863,
-    "kinds": ["bug"],
-    "sigs": ["scheduling", "storage"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "73891": {
-    "commit": "fe37733a124766ee1da11cf1a173488788845cb2",
-    "text": "Kubelet plugin registration now has retry and exponential backoff logic for when registration of plugins (like CSI or device plugin) fail.",
-    "markdown": "Kubelet plugin registration now has retry and exponential backoff logic for when registration of plugins (like CSI or device plugin) fail. ([#73891](https://github.com/kubernetes/kubernetes/pull/73891), [@taragu](https://github.com/taragu)) Courtesy of SIG Node, and SIG Storage",
-    "author": "taragu",
-    "author_url": "https://github.com/taragu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73891",
-    "pr_number": 73891,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node", "storage"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "74027": {
-    "commit": "74743793f2b4de7a2c5e02478ee9dcf7a360ed0a",
-    "text": "The metric `kube_proxy_sync_proxy_rules_last_timestamp_seconds` is now available, indicating the last time that kube-proxy successfully applied proxying rules.",
-    "markdown": "The metric `kube_proxy_sync_proxy_rules_last_timestamp_seconds` is now available, indicating the last time that kube-proxy successfully applied proxying rules. ([#74027](https://github.com/kubernetes/kubernetes/pull/74027), [@squeed](https://github.com/squeed)) Courtesy of SIG Instrumentation, and SIG Network",
-    "author": "squeed",
-    "author_url": "https://github.com/squeed",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74027",
-    "pr_number": 74027,
-    "kinds": ["feature"],
-    "sigs": ["instrumentation", "network"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "74074": {
-    "commit": "107595eb5e2da6f79a07fb8b5900deec7f063141",
-    "text": "Introduce API for watch bookmark events.\nIntroduce Alpha field `AllowWatchBookmarks` in ListOptions for requesting watch bookmarks from apiserver. The implementation in apiserver is hidden behind feature gate `WatchBookmark` (currently in Alpha stage).",
-    "markdown": "Introduce API for watch bookmark events.\nIntroduce Alpha field `AllowWatchBookmarks` in ListOptions for requesting watch bookmarks from apiserver. The implementation in apiserver is hidden behind feature gate `WatchBookmark` (currently in Alpha stage). ([#74074](https://github.com/kubernetes/kubernetes/pull/74074), [@wojtek-t](https://github.com/wojtek-t))",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74074",
-    "pr_number": 74074,
-    "areas": ["apiserver"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "scalability"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "74159": {
-    "commit": "7d15d41eb87f8e6977a8947efe3edf7a75c1d01d",
-    "text": "Support parsing more v1.Taint forms. `key:effect`, `key=:effect-` are now accepted.",
-    "markdown": "Support parsing more v1.Taint forms. `key:effect`, `key=:effect-` are now accepted. ([#74159](https://github.com/kubernetes/kubernetes/pull/74159), [@dlipovetsky](https://github.com/dlipovetsky))",
-    "author": "dlipovetsky",
-    "author_url": "https://github.com/dlipovetsky",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74159",
-    "pr_number": 74159,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "74321": {
-    "commit": "ec64aef25f2fc8c7f3093d16566e6d2e40b61e4e",
-    "text": "watch can now be enabled  for events using the flag --watch-cache-sizes on kube-apiserver",
-    "markdown": "watch can now be enabled  for events using the flag --watch-cache-sizes on kube-apiserver ([#74321](https://github.com/kubernetes/kubernetes/pull/74321), [@yastij](https://github.com/yastij)) Courtesy of SIG API Machinery",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74321",
-    "pr_number": 74321,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "74610": {
-    "commit": "98b6f359992e7d7e6de77aa25ba3315586a8f6b2",
-    "text": "StorageObjectInUseProtection admission plugin is additionally enabled by default.\nSo default enabled admission plugins are now `NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,StorageObjectInUseProtection`. Please note that if you previously had not set the `--admission-control` flag, your cluster behavior may change (to be more standard).",
-    "markdown": "StorageObjectInUseProtection admission plugin is additionally enabled by default.\nSo default enabled admission plugins are now `NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,StorageObjectInUseProtection`. Please note that if you previously had not set the `--admission-control` flag, your cluster behavior may change (to be more standard). ([#74610](https://github.com/kubernetes/kubernetes/pull/74610), [@oomichi](https://github.com/oomichi))",
-    "author": "oomichi",
-    "author_url": "https://github.com/oomichi",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74610",
-    "pr_number": 74610,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "storage"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "74614": {
-    "commit": "6c42729c2c01b2792fa5f5c95d825645eaba2571",
-    "text": "Add  NonPrempting field to the PriorityClass.",
-    "markdown": "Add  NonPrempting field to the PriorityClass. ([#74614](https://github.com/kubernetes/kubernetes/pull/74614), [@denkensk](https://github.com/denkensk)) Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG CLI, SIG Node, SIG Scheduling, and SIG Testing",
-    "author": "denkensk",
-    "author_url": "https://github.com/denkensk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74614",
-    "pr_number": 74614,
-    "areas": ["apiserver", "kubectl", "kubelet", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "auth", "cli", "node", "scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "74690": {
-    "commit": "dc2c9e8bd522d9afe19a1da7e3aa56dca450a024",
-    "text": "Override protocol between etcd server and kube-apiserver on master with HTTPS instead HTTP when mTLS is enabled in GCE",
-    "markdown": "Override protocol between etcd server and kube-apiserver on master with HTTPS instead HTTP when mTLS is enabled in GCE ([#74690](https://github.com/kubernetes/kubernetes/pull/74690), [@wenjiaswe](https://github.com/wenjiaswe))",
-    "author": "wenjiaswe",
-    "author_url": "https://github.com/wenjiaswe",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74690",
-    "pr_number": 74690,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "74692": {
-    "commit": "009f7a07ff4f793c56c0d66d213c54998624f31e",
-    "text": "refactor AWS NLB securityGroup handling",
-    "markdown": "refactor AWS NLB securityGroup handling ([#74692](https://github.com/kubernetes/kubernetes/pull/74692), [@M00nF1sh](https://github.com/M00nF1sh))",
-    "author": "M00nF1sh",
-    "author_url": "https://github.com/M00nF1sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74692",
-    "pr_number": 74692,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.0"
-  },
-  "74747": {
-    "commit": "a8cbb2250624524c8f3bba33f0ec09e8a252c26b",
-    "text": "Fixed a potential deadlock in resource quota controller\nEnabled recording partial usage info for quota objects specifying multiple resources, when only some of the resources' usage can be determined.",
-    "markdown": "Fixed a potential deadlock in resource quota controller\nEnabled recording partial usage info for quota objects specifying multiple resources, when only some of the resources' usage can be determined. ([#74747](https://github.com/kubernetes/kubernetes/pull/74747), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74747",
-    "pr_number": 74747,
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery", "apps"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "74825": {
-    "commit": "8b7e777fe8843419db21293759be861b5a4e682b",
-    "text": "Windows support for preserving the destination IP as the VIP when loadbalancing with DSR.",
-    "markdown": "Windows support for preserving the destination IP as the VIP when loadbalancing with DSR. ([#74825](https://github.com/kubernetes/kubernetes/pull/74825), [@ksubrmnn](https://github.com/ksubrmnn)) Courtesy of SIG Network, and SIG Windows",
-    "author": "ksubrmnn",
-    "author_url": "https://github.com/ksubrmnn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74825",
-    "pr_number": 74825,
-    "areas": ["dependency"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["network", "windows"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "74840": {
-    "commit": "fa833a1e33d45b6438b6c2afa45ff252c5aa582b",
-    "text": "Packets considered INVALID by conntrack are now dropped. In particular, this fixes\na problem where spurious retransmits in a long-running TCP connection to a service\nIP could result in the connection being closed with the error \"Connection reset by\npeer\"",
-    "markdown": "Packets considered INVALID by conntrack are now dropped. In particular, this fixes\na problem where spurious retransmits in a long-running TCP connection to a service\nIP could result in the connection being closed with the error \"Connection reset by\npeer\" ([#74840](https://github.com/kubernetes/kubernetes/pull/74840), [@anfernee](https://github.com/anfernee))",
-    "author": "anfernee",
-    "author_url": "https://github.com/anfernee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74840",
-    "pr_number": 74840,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.15.0"
-  },
-  "74855": {
-    "commit": "eb65eacfcab4b53eb65953b9f59a00e3e2b5a909",
-    "text": "- Added port configuration to Admission webhook configuration service reference.\n- Added port configuration to AuditSink webhook configuration service reference.\n- Added port configuration to CRD Conversion webhook configuration service reference.\n- Added port configuration to kube-aggregator service reference.",
-    "markdown": "- Added port configuration to Admission webhook configuration service reference.\n- Added port configuration to AuditSink webhook configuration service reference.\n- Added port configuration to CRD Conversion webhook configuration service reference.\n- Added port configuration to kube-aggregator service reference. ([#74855](https://github.com/kubernetes/kubernetes/pull/74855), [@mbohlool](https://github.com/mbohlool))",
-    "author": "mbohlool",
-    "author_url": "https://github.com/mbohlool",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74855",
-    "pr_number": 74855,
-    "areas": ["apiserver", "dependency", "release-eng", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "auth", "release", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "74856": {
-    "commit": "d5245b9563d1c3e9c67bc91ca1aa0d0210da2e13",
-    "text": "Remove extra pod creation expections when daemonset fails to create pods in batches.",
-    "markdown": "Remove extra pod creation expections when daemonset fails to create pods in batches. ([#74856](https://github.com/kubernetes/kubernetes/pull/74856), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74856",
-    "pr_number": 74856,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "74877": {
-    "commit": "d920430ce4a9140102d06664c6aa1cd6792cab05",
-    "text": "`k8s.io/kubernetes` and published components (like `k8s.io/client-go` and `k8s.io/api`) now publish go module files containing dependency version information. See http://git.k8s.io/client-go/INSTALL.md\u0026#35;go-modules for details on consuming `k8s.io/client-go` using go modules.",
-    "markdown": "`k8s.io/kubernetes` and published components (like `k8s.io/client-go` and `k8s.io/api`) now publish go module files containing dependency version information. See http://git.k8s.io/client-go/INSTALL.md\u0026#35;go-modules for details on consuming `k8s.io/client-go` using go modules. ([#74877](https://github.com/kubernetes/kubernetes/pull/74877), [@liggitt](https://github.com/liggitt)) Courtesy of SIG API Machinery, SIG CLI, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Node, and SIG Release",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74877",
-    "pr_number": 74877,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "release-eng"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "node", "release"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "74889": {
-    "commit": "657951c3ed79ac43191ed0ef1368a61795b1a572",
-    "text": "Users may now execute `get-kube-binaries.sh` to request a client for an OS/Arch unlike the one of the host on which the script is invoked.",
-    "markdown": "Users may now execute `get-kube-binaries.sh` to request a client for an OS/Arch unlike the one of the host on which the script is invoked. ([#74889](https://github.com/kubernetes/kubernetes/pull/74889), [@akutz](https://github.com/akutz)) Courtesy of SIG Cluster Lifecycle",
-    "author": "akutz",
-    "author_url": "https://github.com/akutz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74889",
-    "pr_number": 74889,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "74910": {
-    "commit": "4f08ea91df235394374a616b9f6a7e3511db0705",
-    "text": "Add TLS termination support for NLB",
-    "markdown": "Add TLS termination support for NLB ([#74910](https://github.com/kubernetes/kubernetes/pull/74910), [@M00nF1sh](https://github.com/M00nF1sh)) Courtesy of SIG Cloud Provider",
-    "author": "M00nF1sh",
-    "author_url": "https://github.com/M00nF1sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74910",
-    "pr_number": 74910,
-    "areas": ["cloudprovider"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "74942": {
-    "commit": "8ec6167f61463cbc6874514b02d0137d51b23542",
-    "text": "Migrate oom watcher not relying on cAdviosr's API any more",
-    "markdown": "Migrate oom watcher not relying on cAdviosr's API any more ([#74942](https://github.com/kubernetes/kubernetes/pull/74942), [@WanLinghao](https://github.com/WanLinghao)) Courtesy of SIG Node",
-    "author": "WanLinghao",
-    "author_url": "https://github.com/WanLinghao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74942",
-    "pr_number": 74942,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "75062": {
-    "commit": "055061637a465fd2333431f2361c9f915bdf951d",
-    "text": "Allow to read OpenStack user credentials from a secret instead of a local config file.",
-    "markdown": "Allow to read OpenStack user credentials from a secret instead of a local config file. ([#75062](https://github.com/kubernetes/kubernetes/pull/75062), [@Fedosin](https://github.com/Fedosin)) Courtesy of SIG Cloud Provider, and SIG Storage",
-    "author": "Fedosin",
-    "author_url": "https://github.com/Fedosin",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75062",
-    "pr_number": 75062,
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider", "storage"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "75072": {
-    "commit": "0118630abf4f47d17770f08e7aad862107d2ce27",
-    "text": "Add name validation for dynamic client methods in client-go",
-    "markdown": "Add name validation for dynamic client methods in client-go ([#75072](https://github.com/kubernetes/kubernetes/pull/75072), [@lblackstone](https://github.com/lblackstone))",
-    "author": "lblackstone",
-    "author_url": "https://github.com/lblackstone",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75072",
-    "pr_number": 75072,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "75109": {
-    "commit": "8a822b0fcf394e58cb6406d4194eb5237d3a333f",
-    "text": "Allow init container to get its own field value as environment variable values(downwardAPI spport)",
-    "markdown": "Allow init container to get its own field value as environment variable values(downwardAPI spport) ([#75109](https://github.com/kubernetes/kubernetes/pull/75109), [@yuchengwu](https://github.com/yuchengwu))",
-    "author": "yuchengwu",
-    "author_url": "https://github.com/yuchengwu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75109",
-    "pr_number": 75109,
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.15.0"
-  },
-  "75179": {
-    "commit": "9c53f24012a0c05662a1b5626cce0271bbe881e4",
-    "text": "kubeadm: Support for deprecated old kubeadm v1alpha3 config is totally removed.",
-    "markdown": "kubeadm: Support for deprecated old kubeadm v1alpha3 config is totally removed. ([#75179](https://github.com/kubernetes/kubernetes/pull/75179), [@rosti](https://github.com/rosti))",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75179",
-    "pr_number": 75179,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "75213": {
-    "commit": "8da0c4af29126097f39bfa57067707e76e8923d1",
-    "text": "Reset exponential backoff when storage operation changes",
-    "markdown": "Reset exponential backoff when storage operation changes ([#75213](https://github.com/kubernetes/kubernetes/pull/75213), [@gnufied](https://github.com/gnufied))",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75213",
-    "pr_number": 75213,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.15.0"
-  },
-  "75278": {
-    "commit": "097d58523a095cd1b2a4dc44273eb46234740a98",
-    "text": "UpdateContainerResources is no longer recorded as a `container_status` operation. It now uses the label `update_container`",
-    "markdown": "UpdateContainerResources is no longer recorded as a `container_status` operation. It now uses the label `update_container` ([#75278](https://github.com/kubernetes/kubernetes/pull/75278), [@Nessex](https://github.com/Nessex))",
-    "author": "Nessex",
-    "author_url": "https://github.com/Nessex",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75278",
-    "pr_number": 75278,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.15.0"
-  },
-  "75279": {
-    "commit": "5229bce04fc7d632f9349480267807a1d41fedf8",
-    "text": "Replace *_admission_latencies_milliseconds_summary and *_admission_latencies_milliseconds metrics due to reporting wrong unit (was labelled milliseconds, but reported seconds), and multiple naming guideline violations (units should be in base units and \"duration\" is the best practice labelling to measure the time a request takes). Please convert to use *_admission_duration_seconds and *_admission_duration_seconds_summary, these now report the unit as described, and follow the instrumentation best practices.",
-    "markdown": "Replace *_admission_latencies_milliseconds_summary and *_admission_latencies_milliseconds metrics due to reporting wrong unit (was labelled milliseconds, but reported seconds), and multiple naming guideline violations (units should be in base units and \"duration\" is the best practice labelling to measure the time a request takes). Please convert to use *_admission_duration_seconds and *_admission_duration_seconds_summary, these now report the unit as described, and follow the instrumentation best practices. ([#75279](https://github.com/kubernetes/kubernetes/pull/75279), [@danielqsj](https://github.com/danielqsj))",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75279",
-    "pr_number": 75279,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "75283": {
-    "commit": "88dc966a8a41fcc694278d7c87114233d2442551",
-    "text": "[IPVS] Allow for transparent kube-proxy restarts",
-    "markdown": "[IPVS] Allow for transparent kube-proxy restarts ([#75283](https://github.com/kubernetes/kubernetes/pull/75283), [@lbernail](https://github.com/lbernail))",
-    "author": "lbernail",
-    "author_url": "https://github.com/lbernail",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75283",
-    "pr_number": 75283,
-    "areas": ["ipvs"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.15.0"
-  },
-  "75295": {
-    "commit": "59140d647449a41ecbe465fec3ec7018f85c6cff",
-    "text": "[IPVS] Introduces flag ipvs-strict-arp to configure stricter ARP sysctls, defaulting to false to preserve existing behaviors. This was enabled by default in 1.13.0, which impacted a few CNI plugins.",
-    "markdown": "[IPVS] Introduces flag ipvs-strict-arp to configure stricter ARP sysctls, defaulting to false to preserve existing behaviors. This was enabled by default in 1.13.0, which impacted a few CNI plugins. ([#75295](https://github.com/kubernetes/kubernetes/pull/75295), [@lbernail](https://github.com/lbernail))",
-    "author": "lbernail",
-    "author_url": "https://github.com/lbernail",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75295",
-    "pr_number": 75295,
-    "areas": ["ipvs"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["network"],
-    "release_version": "1.15.0"
-  },
-  "75362": {
-    "commit": "da018a6bfa2214c7068644ba48d25939e44b9db5",
-    "text": "[stackdriver addon] Bump prometheus-to-sd to v0.5.0 to pick up security fixes.\n[fluentd-gcp addon] Bump fluentd-gcp-scaler to v0.5.1 to pick up security fixes.\n[fluentd-gcp addon] Bump event-exporter to v0.2.4 to pick up security fixes.\n[fluentd-gcp addon] Bump prometheus-to-sd to v0.5.0 to pick up security fixes.\n[metatada-proxy addon] Bump prometheus-to-sd v0.5.0 to pick up security fixes.",
-    "markdown": "[stackdriver addon] Bump prometheus-to-sd to v0.5.0 to pick up security fixes.\n[fluentd-gcp addon] Bump fluentd-gcp-scaler to v0.5.1 to pick up security fixes.\n[fluentd-gcp addon] Bump event-exporter to v0.2.4 to pick up security fixes.\n[fluentd-gcp addon] Bump prometheus-to-sd to v0.5.0 to pick up security fixes.\n[metatada-proxy addon] Bump prometheus-to-sd v0.5.0 to pick up security fixes. ([#75362](https://github.com/kubernetes/kubernetes/pull/75362), [@serathius](https://github.com/serathius))",
-    "author": "serathius",
-    "author_url": "https://github.com/serathius",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75362",
-    "pr_number": 75362,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "gcp", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "75376": {
-    "commit": "ac6b1a1fc514e2b32c080b0de5da8c627b42b279",
-    "text": "Fix some potential deadlocks and file descriptor leaking for inotify watches.",
-    "markdown": "Fix some potential deadlocks and file descriptor leaking for inotify watches. ([#75376](https://github.com/kubernetes/kubernetes/pull/75376), [@cpuguy83](https://github.com/cpuguy83))",
-    "author": "cpuguy83",
-    "author_url": "https://github.com/cpuguy83",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75376",
-    "pr_number": 75376,
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.15.0"
-  },
-  "75389": {
-    "commit": "c79fbabf234bea36f7b870da8e763c542c804be0",
-    "text": "Paginate requests from the kube-apiserver watch cache to etcd in chunks.\nPaginate reflector init and resync List calls that are not served by watch cache.",
-    "markdown": "Paginate requests from the kube-apiserver watch cache to etcd in chunks.\nPaginate reflector init and resync List calls that are not served by watch cache. ([#75389](https://github.com/kubernetes/kubernetes/pull/75389), [@jpbetz](https://github.com/jpbetz)) Courtesy of SIG API Machinery, SIG Apps, and SIG Testing",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75389",
-    "pr_number": 75389,
-    "areas": ["apiserver", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "apps", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "75405": {
-    "commit": "ec88ec272ccdc824c56684628b113f09b873a9b2",
-    "text": "Fix bug where cloud-controller-manager initializes nodes multiple times",
-    "markdown": "Fix bug where cloud-controller-manager initializes nodes multiple times ([#75405](https://github.com/kubernetes/kubernetes/pull/75405), [@tghartland](https://github.com/tghartland))",
-    "author": "tghartland",
-    "author_url": "https://github.com/tghartland",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75405",
-    "pr_number": 75405,
-    "areas": ["cloudprovider", "controller-manager"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "cloud-provider", "scalability"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "75459": {
-    "commit": "ae2a1628f66f68a59625283f427f0a794ad0d53a",
-    "text": "API changes and deprecating the use of special annotations for Windows GMSA support (version beta)",
-    "markdown": "API changes and deprecating the use of special annotations for Windows GMSA support (version beta) ([#75459](https://github.com/kubernetes/kubernetes/pull/75459), [@wk8](https://github.com/wk8)) Courtesy of SIG API Machinery, SIG Apps, SIG Node, SIG Testing, and SIG Windows",
-    "author": "wk8",
-    "author_url": "https://github.com/wk8",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75459",
-    "pr_number": 75459,
-    "areas": ["kubelet", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "node", "testing", "windows"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "75479": {
-    "commit": "89ae84d478d418b1bf4b44c4544020ab110c5bf5",
-    "text": "Kubelet will attempt to use wincat.exe in the pause container for port forwarding when running on Windows",
-    "markdown": "Kubelet will attempt to use wincat.exe in the pause container for port forwarding when running on Windows ([#75479](https://github.com/kubernetes/kubernetes/pull/75479), [@benmoss](https://github.com/benmoss)) Courtesy of SIG Node, and SIG Windows",
-    "author": "benmoss",
-    "author_url": "https://github.com/benmoss",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75479",
-    "pr_number": 75479,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node", "windows"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "75480": {
-    "commit": "164fad5e3f447e3eb55154cc58bcbe4b3016d129",
-    "text": "Update Cluster Autoscaler to 1.14.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.14.0",
-    "markdown": "Update Cluster Autoscaler to 1.14.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.14.0 ([#75480](https://github.com/kubernetes/kubernetes/pull/75480), [@losipiuk](https://github.com/losipiuk))",
-    "author": "losipiuk",
-    "author_url": "https://github.com/losipiuk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75480",
-    "pr_number": 75480,
-    "kinds": ["bug"],
-    "sigs": ["autoscaling", "cluster-lifecycle"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "75483": {
-    "commit": "483cd0db09e9af9e7a37f59ed90c33264eee0740",
-    "text": "In the 'kubectl describe' output, the fields with names containing special characters are displayed as-is without any pretty formatting. ",
-    "markdown": "In the 'kubectl describe' output, the fields with names containing special characters are displayed as-is without any pretty formatting.  ([#75483](https://github.com/kubernetes/kubernetes/pull/75483), [@gsadhani](https://github.com/gsadhani))",
-    "author": "gsadhani",
-    "author_url": "https://github.com/gsadhani",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75483",
-    "pr_number": 75483,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.15.0"
-  },
-  "75487": {
-    "commit": "e6272b887b81a62e6f06b7fac4b3b61d1c8bf657",
-    "text": "kubeadm: fix the machine readability of \"kubeadm token create --print-join-command\"",
-    "markdown": "kubeadm: fix the machine readability of \"kubeadm token create --print-join-command\" ([#75487](https://github.com/kubernetes/kubernetes/pull/75487), [@displague](https://github.com/displague))",
-    "author": "displague",
-    "author_url": "https://github.com/displague",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75487",
-    "pr_number": 75487,
-    "areas": ["kubeadm"],
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "75499": {
-    "commit": "73b8011d4744be6465ecc762cc84d3cbc092d1f9",
-    "text": "kubeadm's ignored pre-flight errors can now be configured via InitConfiguration and JoinConfiguration.",
-    "markdown": "kubeadm's ignored pre-flight errors can now be configured via InitConfiguration and JoinConfiguration. ([#75499](https://github.com/kubernetes/kubernetes/pull/75499), [@marccarre](https://github.com/marccarre)) Courtesy of SIG Cluster Lifecycle",
-    "author": "marccarre",
-    "author_url": "https://github.com/marccarre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75499",
-    "pr_number": 75499,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "75501": {
-    "commit": "a93f803f8e400f1d42dc812bc51932ff3b31798a",
-    "text": "scheduler: add metrics to record number of pending pods in different queues",
-    "markdown": "scheduler: add metrics to record number of pending pods in different queues ([#75501](https://github.com/kubernetes/kubernetes/pull/75501), [@Huang-Wei](https://github.com/Huang-Wei)) Courtesy of SIG Apps, and SIG Scheduling",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75501",
-    "pr_number": 75501,
-    "kinds": ["feature"],
-    "sigs": ["apps", "scheduling"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "75515": {
-    "commit": "9c973c6d2c33e88521ebcebec2fdd9cbddccd857",
-    "text": "Support vSphere SAML token auth when using Zones",
-    "markdown": "Support vSphere SAML token auth when using Zones ([#75515](https://github.com/kubernetes/kubernetes/pull/75515), [@dougm](https://github.com/dougm))",
-    "author": "dougm",
-    "author_url": "https://github.com/dougm",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75515",
-    "pr_number": 75515,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "vmware"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "75531": {
-    "commit": "4b3eb600811339fe1b3972672edafa1fe10309ec",
-    "text": "CRI API will now be available in the kubernetes/cri-api repository",
-    "markdown": "CRI API will now be available in the kubernetes/cri-api repository ([#75531](https://github.com/kubernetes/kubernetes/pull/75531), [@dims](https://github.com/dims)) Courtesy of SIG Network, SIG Node, SIG Release, and SIG Testing",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75531",
-    "pr_number": 75531,
-    "areas": ["kubelet", "release-eng", "test"],
-    "kinds": ["feature"],
-    "sigs": ["network", "node", "release", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "75580": {
-    "commit": "2fa5be16334dfaca05049798f2cb73ac7979a4b6",
-    "text": "Support specify the Resource Group of Route Table when update Pod network route (Azure)",
-    "markdown": "Support specify the Resource Group of Route Table when update Pod network route (Azure) ([#75580](https://github.com/kubernetes/kubernetes/pull/75580), [@suker200](https://github.com/suker200)) Courtesy of SIG Cloud Provider",
-    "author": "suker200",
-    "author_url": "https://github.com/suker200",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75580",
-    "pr_number": 75580,
-    "areas": ["cloudprovider"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "75587": {
-    "commit": "dabeb20a1ae379a27359ceb1bc77ff49dbd6db12",
-    "text": "The AWS credential provider can now obtain ECR credentials even without the AWS cloud provider or being on an EC2 instance. Additionally, AWS credential provider caching has been improved to honor the ECR credential timeout.",
-    "markdown": "The AWS credential provider can now obtain ECR credentials even without the AWS cloud provider or being on an EC2 instance. Additionally, AWS credential provider caching has been improved to honor the ECR credential timeout. ([#75587](https://github.com/kubernetes/kubernetes/pull/75587), [@tiffanyfay](https://github.com/tiffanyfay))",
-    "author": "tiffanyfay",
-    "author_url": "https://github.com/tiffanyfay",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75587",
-    "pr_number": 75587,
-    "areas": ["cloudprovider", "kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["aws", "cloud-provider", "node"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "75602": {
-    "commit": "3312da83c9310e9b2b54aa2556f9da06b4bb1394",
-    "text": "The Kubelet now properly requests protobuf objects where they are\nsupported from the apiserver, reducing load in large clusters.",
-    "markdown": "The Kubelet now properly requests protobuf objects where they are\nsupported from the apiserver, reducing load in large clusters. ([#75602](https://github.com/kubernetes/kubernetes/pull/75602), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75602",
-    "pr_number": 75602,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "node", "scalability"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "75622": {
-    "commit": "eee8af93baf00786d129d233d0e80a5e01150808",
-    "text": "StatefulSet controllers no longer force a resync every 30 seconds when nothing has changed.",
-    "markdown": "StatefulSet controllers no longer force a resync every 30 seconds when nothing has changed. ([#75622](https://github.com/kubernetes/kubernetes/pull/75622), [@jonsabo](https://github.com/jonsabo))",
-    "author": "jonsabo",
-    "author_url": "https://github.com/jonsabo",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75622",
-    "pr_number": 75622,
-    "kinds": ["cleanup"],
-    "sigs": ["apps"],
-    "release_version": "1.15.0"
-  },
-  "75646": {
-    "commit": "e6d274233c2a4d02fc927d8a62b44b8b74811382",
-    "text": "kubeadm: implement deletion of multiple bootstrap tokens at once",
-    "markdown": "kubeadm: implement deletion of multiple bootstrap tokens at once ([#75646](https://github.com/kubernetes/kubernetes/pull/75646), [@bart0sh](https://github.com/bart0sh)) Courtesy of SIG Cluster Lifecycle",
-    "author": "bart0sh",
-    "author_url": "https://github.com/bart0sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75646",
-    "pr_number": 75646,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "75658": {
-    "commit": "e3c3f75b5b3d1b236ecc528777cfc1ec9b3cf112",
-    "text": "Update cri-tools to v1.14.0",
-    "markdown": "Update cri-tools to v1.14.0 ([#75658](https://github.com/kubernetes/kubernetes/pull/75658), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75658",
-    "pr_number": 75658,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "75712": {
-    "commit": "16a03518812fa29c87560edd0fea26d299c3d8e2",
-    "text": "Add completed job status in Cronjob event.",
-    "markdown": "Add completed job status in Cronjob event. ([#75712](https://github.com/kubernetes/kubernetes/pull/75712), [@danielqsj](https://github.com/danielqsj)) Courtesy of SIG Apps",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75712",
-    "pr_number": 75712,
-    "kinds": ["feature"],
-    "sigs": ["apps"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "75747": {
-    "commit": "42877480ad9b784956c5d9228ca6e2232778ce5f",
-    "text": "CSI alpha CRDs have been removed",
-    "markdown": "CSI alpha CRDs have been removed ([#75747](https://github.com/kubernetes/kubernetes/pull/75747), [@msau42](https://github.com/msau42))",
-    "author": "msau42",
-    "author_url": "https://github.com/msau42",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75747",
-    "pr_number": 75747,
-    "areas": ["release-eng", "test"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["apps", "cluster-lifecycle", "release", "storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "75750": {
-    "commit": "f25fa0e9e1ce2e275d2a9f468391783b0f04e56e",
-    "text": "Adds a new \"storage_operation_status_count\" metric for kube-controller-manager and kubelet to count success and error statues.",
-    "markdown": "Adds a new \"storage_operation_status_count\" metric for kube-controller-manager and kubelet to count success and error statues. ([#75750](https://github.com/kubernetes/kubernetes/pull/75750), [@msau42](https://github.com/msau42))",
-    "author": "msau42",
-    "author_url": "https://github.com/msau42",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75750",
-    "pr_number": 75750,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["instrumentation", "storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "75771": {
-    "commit": "fc556f0d9ba59ac47c76e4d60a9ba261e0c622f3",
-    "text": "client-go: The `rest.AnonymousClientConfig(*rest.Config) *rest.Config` helper method no longer copies custom `Transport` and `WrapTransport` fields, because those can be used to inject user credentials.",
-    "markdown": "client-go: The `rest.AnonymousClientConfig(*rest.Config) *rest.Config` helper method no longer copies custom `Transport` and `WrapTransport` fields, because those can be used to inject user credentials. ([#75771](https://github.com/kubernetes/kubernetes/pull/75771), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75771",
-    "pr_number": 75771,
-    "areas": ["client-libraries"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "75781": {
-    "commit": "a0e71b49b6b3eb1ffba097b0c12edd4029fd3363",
-    "text": "Increased verbose level for local openapi aggregation logs to avoid flooding the log during normal operation",
-    "markdown": "Increased verbose level for local openapi aggregation logs to avoid flooding the log during normal operation ([#75781](https://github.com/kubernetes/kubernetes/pull/75781), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75781",
-    "pr_number": 75781,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "75799": {
-    "commit": "861f1fcb2e12c0a7eba5a8e8c894cbed7076ee11",
-    "text": "The Node.Status.Volumes.Attached.DevicePath field is now unset for CSI volumes. Update any external controllers that depend on this field.",
-    "markdown": "The Node.Status.Volumes.Attached.DevicePath field is now unset for CSI volumes. Update any external controllers that depend on this field. ([#75799](https://github.com/kubernetes/kubernetes/pull/75799), [@msau42](https://github.com/msau42)) Courtesy of SIG Storage",
-    "author": "msau42",
-    "author_url": "https://github.com/msau42",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75799",
-    "pr_number": 75799,
-    "kinds": ["cleanup"],
-    "sigs": ["storage"],
-    "action_required": true,
-    "release_version": "1.15.0"
-  },
-  "75847": {
-    "commit": "91f40aec8ad5b992d8057b4fab0fa287ee6bdee9",
-    "text": "kubeadm: preflight checks on external etcd certificates are now skipped when joining a control-plane node with automatic copy of cluster certificates (--certificate-key)",
-    "markdown": "kubeadm: preflight checks on external etcd certificates are now skipped when joining a control-plane node with automatic copy of cluster certificates (--certificate-key) ([#75847](https://github.com/kubernetes/kubernetes/pull/75847), [@fabriziopandini](https://github.com/fabriziopandini))",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75847",
-    "pr_number": 75847,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "75848": {
-    "commit": "cf22c56c7fbeaa166e5e86ebc4c0b75250b9843a",
-    "text": "Implement and update interfaces and skeleton for the scheduling framework.",
-    "markdown": "Implement and update interfaces and skeleton for the scheduling framework. ([#75848](https://github.com/kubernetes/kubernetes/pull/75848), [@bsalamat](https://github.com/bsalamat)) Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "bsalamat",
-    "author_url": "https://github.com/bsalamat",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75848",
-    "pr_number": 75848,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "75849": {
-    "commit": "d5dbc0019123f42c678a1f012a068941d56087c8",
-    "text": "Add ListPager.EachListItem utility function to client-go to enable incremental processing of chunked list responses",
-    "markdown": "Add ListPager.EachListItem utility function to client-go to enable incremental processing of chunked list responses ([#75849](https://github.com/kubernetes/kubernetes/pull/75849), [@jpbetz](https://github.com/jpbetz)) Courtesy of SIG API Machinery, and SIG Apps",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75849",
-    "pr_number": 75849,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "apps"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "75853": {
-    "commit": "f8d0b21b980f33e218b7e1f01a13bf3bd90713f9",
-    "text": "Use stdlib to log stack trace when a panic occurs",
-    "markdown": "Use stdlib to log stack trace when a panic occurs ([#75853](https://github.com/kubernetes/kubernetes/pull/75853), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75853",
-    "pr_number": 75853,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "75855": {
-    "commit": "6fef6e8613c0b4fec54b129e9c8b74daff253296",
-    "text": "GCE Windows nodes will rely solely on kubernetes and kube-proxy (and not the GCE agent) for network address management.",
-    "markdown": "GCE Windows nodes will rely solely on kubernetes and kube-proxy (and not the GCE agent) for network address management. ([#75855](https://github.com/kubernetes/kubernetes/pull/75855), [@pjh](https://github.com/pjh))",
-    "author": "pjh",
-    "author_url": "https://github.com/pjh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75855",
-    "pr_number": 75855,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "75857": {
-    "commit": "2792f1a24e1d9e6aaccde3661cc6a25c5d728d88",
-    "text": "Support both JSON and YAML for scheduler configuration.",
-    "markdown": "Support both JSON and YAML for scheduler configuration. ([#75857](https://github.com/kubernetes/kubernetes/pull/75857), [@danielqsj](https://github.com/danielqsj)) Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75857",
-    "pr_number": 75857,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "75866": {
-    "commit": "2c675818ed46579ca91ed88a4222cc87c7328a16",
-    "text": "Add --image-repository flag to \"kubeadm config images\".",
-    "markdown": "Add --image-repository flag to \"kubeadm config images\". ([#75866](https://github.com/kubernetes/kubernetes/pull/75866), [@jmkeyes](https://github.com/jmkeyes)) Courtesy of SIG Cluster Lifecycle",
-    "author": "jmkeyes",
-    "author_url": "https://github.com/jmkeyes",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75866",
-    "pr_number": 75866,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "75887": {
-    "commit": "6681c127d308f4bb4ab92435427e7f51d6c34f67",
-    "text": "Fixes a regression proxying responses from aggregated API servers which could cause watch requests to hang until the first event was received",
-    "markdown": "Fixes a regression proxying responses from aggregated API servers which could cause watch requests to hang until the first event was received ([#75887](https://github.com/kubernetes/kubernetes/pull/75887), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75887",
-    "pr_number": 75887,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "75900": {
-    "commit": "8f304510c5345cc0c5132c5219304583a595b139",
-    "text": "kubeadm: fix \"upgrade plan\" not defaulting to a \"stable\" version if no version argument is passed",
-    "markdown": "kubeadm: fix \"upgrade plan\" not defaulting to a \"stable\" version if no version argument is passed ([#75900](https://github.com/kubernetes/kubernetes/pull/75900), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75900",
-    "pr_number": 75900,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "75920": {
-    "commit": "3d871df19a1686c1f9a1e44ec0aab207d61805be",
-    "text": "Add --sort-by option to kubectl top command",
-    "markdown": "Add --sort-by option to kubectl top command ([#75920](https://github.com/kubernetes/kubernetes/pull/75920), [@artmello](https://github.com/artmello)) Courtesy of SIG CLI",
-    "author": "artmello",
-    "author_url": "https://github.com/artmello",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75920",
-    "pr_number": 75920,
-    "areas": ["kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "75969": {
-    "commit": "c3cc31797a4934b29fc1244a5828ae02e3c4b68d",
-    "text": " `StubDomains` and `Upstreamnameserver` which contains a service name will be omitted while translating to the equivalent CoreDNS config.",
-    "markdown": " `StubDomains` and `Upstreamnameserver` which contains a service name will be omitted while translating to the equivalent CoreDNS config. ([#75969](https://github.com/kubernetes/kubernetes/pull/75969), [@rajansandeep](https://github.com/rajansandeep))",
-    "author": "rajansandeep",
-    "author_url": "https://github.com/rajansandeep",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75969",
-    "pr_number": 75969,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "75971": {
-    "commit": "a3ae6c43c266ebfb7496430f2938993c6de81440",
-    "text": "Resource list requests for PartialObjectMetadata now correctly return list metadata like the resourceVersion and the continue token.",
-    "markdown": "Resource list requests for PartialObjectMetadata now correctly return list metadata like the resourceVersion and the continue token. ([#75971](https://github.com/kubernetes/kubernetes/pull/75971), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75971",
-    "pr_number": 75971,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "75985": {
-    "commit": "16db83b257af11370b575c362e79ee5d7f1b09ff",
-    "text": "Tolerations with the same key and effect will be merged into one which has the value of the latest toleration for best effort pods.",
-    "markdown": "Tolerations with the same key and effect will be merged into one which has the value of the latest toleration for best effort pods. ([#75985](https://github.com/kubernetes/kubernetes/pull/75985), [@ravisantoshgudimetla](https://github.com/ravisantoshgudimetla))",
-    "author": "ravisantoshgudimetla",
-    "author_url": "https://github.com/ravisantoshgudimetla",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75985",
-    "pr_number": 75985,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.15.0"
-  },
-  "75986": {
-    "commit": "88992ba185c1c9205eef93ed9c558a70bd9da641",
-    "text": "Reduce event spam for AttachVolume storage operation",
-    "markdown": "Reduce event spam for AttachVolume storage operation ([#75986](https://github.com/kubernetes/kubernetes/pull/75986), [@mucahitkurt](https://github.com/mucahitkurt))",
-    "author": "mucahitkurt",
-    "author_url": "https://github.com/mucahitkurt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75986",
-    "pr_number": 75986,
-    "kinds": ["cleanup"],
-    "sigs": ["storage"],
-    "release_version": "1.15.0"
-  },
-  "75987": {
-    "commit": "8136cfc512754e0b3a943acc2a0d12aa3903e019",
-    "text": "The `series.state` field in the events.k8s.io/v1beta1 Event API is deprecated and will be removed in v1.18",
-    "markdown": "The `series.state` field in the events.k8s.io/v1beta1 Event API is deprecated and will be removed in v1.18 ([#75987](https://github.com/kubernetes/kubernetes/pull/75987), [@yastij](https://github.com/yastij))",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75987",
-    "pr_number": 75987,
-    "kinds": ["api-change"],
-    "sigs": ["apps", "instrumentation", "scalability"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "75990": {
-    "commit": "4036097edb8c77ca553d0b6901926db7b6447798",
-    "text": "consume the AWS region list from the AWS SDK instead of a hard-coded list in the cloud provider",
-    "markdown": "consume the AWS region list from the AWS SDK instead of a hard-coded list in the cloud provider ([#75990](https://github.com/kubernetes/kubernetes/pull/75990), [@mcrute](https://github.com/mcrute))",
-    "author": "mcrute",
-    "author_url": "https://github.com/mcrute",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75990",
-    "pr_number": 75990,
-    "areas": ["cloudprovider"],
-    "kinds": ["cleanup"],
-    "sigs": ["aws", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "75993": {
-    "commit": "1fb8ed12a2d1d50a890f24c67c5fc0d37d4ffb04",
-    "text": "Adding ListMeta.RemainingItemCount. When responding a LIST request, if the server has more data available, and if the request does not contain label selectors or field selectors, the server sets the ListOptions.RemainingItemCount to the number of remaining objects.",
-    "markdown": "Adding ListMeta.RemainingItemCount. When responding a LIST request, if the server has more data available, and if the request does not contain label selectors or field selectors, the server sets the ListOptions.RemainingItemCount to the number of remaining objects. ([#75993](https://github.com/kubernetes/kubernetes/pull/75993), [@caesarxuchao](https://github.com/caesarxuchao))",
-    "author": "caesarxuchao",
-    "author_url": "https://github.com/caesarxuchao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75993",
-    "pr_number": 75993,
-    "areas": ["apiserver", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76014": {
-    "commit": "4c4c378d99c207f026a3e9b2c1fac625ce613535",
-    "text": "improve `kubectl auth can-i` command by warning users when they try access resource out of scope",
-    "markdown": "improve `kubectl auth can-i` command by warning users when they try access resource out of scope ([#76014](https://github.com/kubernetes/kubernetes/pull/76014), [@WanLinghao](https://github.com/WanLinghao))",
-    "author": "WanLinghao",
-    "author_url": "https://github.com/WanLinghao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76014",
-    "pr_number": 76014,
-    "areas": ["kubectl", "test"],
-    "kinds": ["bug"],
-    "sigs": ["auth", "cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76039": {
-    "commit": "41691a031a5dd1b8e8912061dcb5efe488167115",
-    "text": "Ensures the conformance test image saves results before exiting when ginkgo returns non-zero value.",
-    "markdown": "Ensures the conformance test image saves results before exiting when ginkgo returns non-zero value. ([#76039](https://github.com/kubernetes/kubernetes/pull/76039), [@johnSchnake](https://github.com/johnSchnake))",
-    "author": "johnSchnake",
-    "author_url": "https://github.com/johnSchnake",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76039",
-    "pr_number": 76039,
-    "areas": ["conformance"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "release", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76060": {
-    "commit": "8f8d9e2c726e98225123c032462f3fd155d9bb6e",
-    "text": "Fixes bug in DaemonSetController causing it to stop processing some DaemonSets for 5 minutes after node removal.",
-    "markdown": "Fixes bug in DaemonSetController causing it to stop processing some DaemonSets for 5 minutes after node removal. ([#76060](https://github.com/kubernetes/kubernetes/pull/76060), [@krzysztof-jastrzebski](https://github.com/krzysztof-jastrzebski))",
-    "author": "krzysztof-jastrzebski",
-    "author_url": "https://github.com/krzysztof-jastrzebski",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76060",
-    "pr_number": 76060,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.15.0"
-  },
-  "76062": {
-    "commit": "3409a5362aac664594f7e87e23576b230b400730",
-    "text": "Create a new `kubectl rollout restart` command that does a rolling restart of a deployment.",
-    "markdown": "Create a new `kubectl rollout restart` command that does a rolling restart of a deployment. ([#76062](https://github.com/kubernetes/kubernetes/pull/76062), [@apelisse](https://github.com/apelisse)) Courtesy of SIG CLI, and SIG Testing",
-    "author": "apelisse",
-    "author_url": "https://github.com/apelisse",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76062",
-    "pr_number": 76062,
-    "areas": ["kubectl", "test"],
-    "kinds": ["feature"],
-    "sigs": ["cli", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "76063": {
-    "commit": "658c290ed507df073e8a0729471d40676e752aa9",
-    "text": "give users the option to suppress detailed output in integration test",
-    "markdown": "give users the option to suppress detailed output in integration test ([#76063](https://github.com/kubernetes/kubernetes/pull/76063), [@Huang-Wei](https://github.com/Huang-Wei)) Courtesy of SIG Testing",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76063",
-    "pr_number": 76063,
-    "kinds": ["feature"],
-    "sigs": ["testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "76074": {
-    "commit": "b7858e31d416312aab394df7c0d0c42e2c179dce",
-    "text": "Change kubelet probe metrics to counter type.\nThe metrics `prober_probe_result` is replaced by `prober_probe_total`.",
-    "markdown": "Change kubelet probe metrics to counter type.\nThe metrics `prober_probe_result` is replaced by `prober_probe_total`. ([#76074](https://github.com/kubernetes/kubernetes/pull/76074), [@danielqsj](https://github.com/danielqsj)) Courtesy of SIG Instrumentation, and SIG Node",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76074",
-    "pr_number": 76074,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["instrumentation", "node"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "76099": {
-    "commit": "3e7134891f9e9fc4afe3fe28e0fb11721afbe945",
-    "text": "GCE/Windows: disable stackdriver logging agent to prevent node startup failures",
-    "markdown": "GCE/Windows: disable stackdriver logging agent to prevent node startup failures ([#76099](https://github.com/kubernetes/kubernetes/pull/76099), [@yujuhong](https://github.com/yujuhong))",
-    "author": "yujuhong",
-    "author_url": "https://github.com/yujuhong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76099",
-    "pr_number": 76099,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "gcp"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76109": {
-    "commit": "71f4c9a51c60a38f71dae7ab189bf25679accfbb",
-    "text": "kube-proxy no longer automatically cleans up network rules created by running kube-proxy in other modes. If you are switching the mode that kube-proxy is in running in (EG: iptables to IPVS), you will need to run `kube-proxy --cleanup`, or restart the worker node (recommended) before restarting kube-proxy.\n\nIf you are not switching kube-proxy between different modes, this change should not require any action.",
-    "markdown": "kube-proxy no longer automatically cleans up network rules created by running kube-proxy in other modes. If you are switching the mode that kube-proxy is in running in (EG: iptables to IPVS), you will need to run `kube-proxy --cleanup`, or restart the worker node (recommended) before restarting kube-proxy.\n\nIf you are not switching kube-proxy between different modes, this change should not require any action. ([#76109](https://github.com/kubernetes/kubernetes/pull/76109), [@vllry](https://github.com/vllry))",
-    "author": "vllry",
-    "author_url": "https://github.com/vllry",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76109",
-    "pr_number": 76109,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.15.0"
-  },
-  "76161": {
-    "commit": "c082ace102e217846833baace01d80966369ecd8",
-    "text": "`kubectl get -w` now prints custom resource definitions with custom print columns",
-    "markdown": "`kubectl get -w` now prints custom resource definitions with custom print columns ([#76161](https://github.com/kubernetes/kubernetes/pull/76161), [@liggitt](https://github.com/liggitt)) Courtesy of SIG CLI",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76161",
-    "pr_number": 76161,
-    "areas": ["kubectl"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "76188": {
-    "commit": "fad34e404ef690102d2a86b939a8a581e976e5ae",
-    "text": "Expose CSI volume stats via kubelet volume metrics",
-    "markdown": "Expose CSI volume stats via kubelet volume metrics ([#76188](https://github.com/kubernetes/kubernetes/pull/76188), [@humblec](https://github.com/humblec)) Courtesy of SIG Storage",
-    "author": "humblec",
-    "author_url": "https://github.com/humblec",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76188",
-    "pr_number": 76188,
-    "kinds": ["feature"],
-    "sigs": ["storage"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "76211": {
-    "commit": "3e954d3bd30c5293ecc4cfd7f259852875ea7f79",
-    "text": "Fixes a NPD bug on GCI, so that it disables glog writing to files for log-counter",
-    "markdown": "Fixes a NPD bug on GCI, so that it disables glog writing to files for log-counter ([#76211](https://github.com/kubernetes/kubernetes/pull/76211), [@wangzhen127](https://github.com/wangzhen127))",
-    "author": "wangzhen127",
-    "author_url": "https://github.com/wangzhen127",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76211",
-    "pr_number": 76211,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "gcp", "node"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76221": {
-    "commit": "922f3603eae623d7f0e56fee4bf5b05eb7e82c43",
-    "text": "As of Kubernetes 1.15, the SupportNodePidsLimit feature introduced as alpha in Kubernetes 1.14 is now beta, and the ability to utilize it is enabled by default.  It is no longer necessary to set the feature gate `SupportNodePidsLimit=true`.  In all other respects, this functionality behaves as it did in Kubernetes 1.14.",
-    "markdown": "As of Kubernetes 1.15, the SupportNodePidsLimit feature introduced as alpha in Kubernetes 1.14 is now beta, and the ability to utilize it is enabled by default.  It is no longer necessary to set the feature gate `SupportNodePidsLimit=true`.  In all other respects, this functionality behaves as it did in Kubernetes 1.14. ([#76221](https://github.com/kubernetes/kubernetes/pull/76221), [@RobertKrawitz](https://github.com/RobertKrawitz)) Courtesy of SIG Testing",
-    "author": "RobertKrawitz",
-    "author_url": "https://github.com/RobertKrawitz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76221",
-    "pr_number": 76221,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "76238": {
-    "commit": "888b81b638171929d2a7a032bba7386a1cbff9c7",
-    "text": "Default TTL for DNS records in kubernetes zone is changed from 5s to 30s to keep consistent with old dnsmasq based kube-dns. The TTL can be customized with command `kubectl edit -n kube-system configmap/coredns`.",
-    "markdown": "Default TTL for DNS records in kubernetes zone is changed from 5s to 30s to keep consistent with old dnsmasq based kube-dns. The TTL can be customized with command `kubectl edit -n kube-system configmap/coredns`. ([#76238](https://github.com/kubernetes/kubernetes/pull/76238), [@Dieken](https://github.com/Dieken))",
-    "author": "Dieken",
-    "author_url": "https://github.com/Dieken",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76238",
-    "pr_number": 76238,
-    "areas": ["kubeadm", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76243": {
-    "commit": "8a9ed4c91a54b1b29c3b60dad815c372bcfdef76",
-    "text": "2X performance improvement on both required and preferred PodAffinity.",
-    "markdown": "2X performance improvement on both required and preferred PodAffinity. ([#76243](https://github.com/kubernetes/kubernetes/pull/76243), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76243",
-    "pr_number": 76243,
-    "kinds": ["design"],
-    "sigs": ["scheduling"],
-    "release_version": "1.15.0"
-  },
-  "76320": {
-    "commit": "4dc05dd9dff1283984acd77d68d8dce7b7cdfb68",
-    "text": "kube-proxy: HealthzBindAddress and MetricsBindAddress support ipv6 address.",
-    "markdown": "kube-proxy: HealthzBindAddress and MetricsBindAddress support ipv6 address. ([#76320](https://github.com/kubernetes/kubernetes/pull/76320), [@JieJhih](https://github.com/JieJhih))",
-    "author": "JieJhih",
-    "author_url": "https://github.com/JieJhih",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76320",
-    "pr_number": 76320,
-    "kinds": ["cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.15.0"
-  },
-  "76341": {
-    "commit": "d424bedc9b7caa4f153c422299411e76b5e64986",
-    "text": "Fix issue in Portworx volume driver causing controller manager to crash",
-    "markdown": "Fix issue in Portworx volume driver causing controller manager to crash ([#76341](https://github.com/kubernetes/kubernetes/pull/76341), [@harsh-px](https://github.com/harsh-px))",
-    "author": "harsh-px",
-    "author_url": "https://github.com/harsh-px",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76341",
-    "pr_number": 76341,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.15.0"
-  },
-  "76352": {
-    "commit": "ae8a4fcb0d79982c1c8f48be28c34ca7a2ebd6c0",
-    "text": "Reduce GCE log rotation check from 1 hour to every 5 minutes.  Rotation policy is unchanged (new day starts, log file size \u003e 100MB).",
-    "markdown": "Reduce GCE log rotation check from 1 hour to every 5 minutes.  Rotation policy is unchanged (new day starts, log file size \u003e 100MB). ([#76352](https://github.com/kubernetes/kubernetes/pull/76352), [@jpbetz](https://github.com/jpbetz))",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76352",
-    "pr_number": 76352,
-    "areas": ["platform/gce"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "gcp"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76353": {
-    "commit": "134ed2e7031bc91c7b35796553fbcd3ba8dc042f",
-    "text": "Added `CNI_VERSION` and `CNI_SHA1` environment variables in kube-up.sh to configure CNI versions on GCE.",
-    "markdown": "Added `CNI_VERSION` and `CNI_SHA1` environment variables in kube-up.sh to configure CNI versions on GCE. ([#76353](https://github.com/kubernetes/kubernetes/pull/76353), [@Random-Liu](https://github.com/Random-Liu)) Courtesy of SIG Network, and SIG Node",
-    "author": "Random-Liu",
-    "author_url": "https://github.com/Random-Liu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76353",
-    "pr_number": 76353,
-    "kinds": ["feature"],
-    "sigs": ["network", "node"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "76390": {
-    "commit": "3148eb750da9f2259c4105e80a0088acdf1f40cd",
-    "text": "kubeadm: add optional ECDSA support.\nkubeadm still generates RSA keys when deploying a node, but also accepts ECDSA\nkeys if they exist already in the directory specified in --cert-dir option.",
-    "markdown": "kubeadm: add optional ECDSA support.\nkubeadm still generates RSA keys when deploying a node, but also accepts ECDSA\nkeys if they exist already in the directory specified in --cert-dir option. ([#76390](https://github.com/kubernetes/kubernetes/pull/76390), [@rojkov](https://github.com/rojkov)) Courtesy of SIG Cluster Lifecycle",
-    "author": "rojkov",
-    "author_url": "https://github.com/rojkov",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76390",
-    "pr_number": 76390,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "76437": {
-    "commit": "0da72a1ef3a3edf752e931681c171f09d545a79b",
-    "text": "Bump metrics-server to v0.3.2",
-    "markdown": "Bump metrics-server to v0.3.2 ([#76437](https://github.com/kubernetes/kubernetes/pull/76437), [@brett-elliott](https://github.com/brett-elliott))",
-    "author": "brett-elliott",
-    "author_url": "https://github.com/brett-elliott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76437",
-    "pr_number": 76437,
-    "kinds": ["failing-test"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "76474": {
-    "commit": "17fe18bd9cc5652a5dee688255323ce2d07538dd",
-    "text": "Updated klog to 0.3.0",
-    "markdown": "Updated klog to 0.3.0 ([#76474](https://github.com/kubernetes/kubernetes/pull/76474), [@vincepri](https://github.com/vincepri))",
-    "author": "vincepri",
-    "author_url": "https://github.com/vincepri",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76474",
-    "pr_number": 76474,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "node"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76505": {
-    "commit": "299aa5d7a62188e27d7a6cd8287b2411574587ba",
-    "text": "kubeadm: Fix a bug where if couple of CRIs are installed a user override of the CRI during join (via kubeadm join --cri-socket ...) is ignored and kubeadm bails out with an error",
-    "markdown": "kubeadm: Fix a bug where if couple of CRIs are installed a user override of the CRI during join (via kubeadm join --cri-socket ...) is ignored and kubeadm bails out with an error ([#76505](https://github.com/kubernetes/kubernetes/pull/76505), [@rosti](https://github.com/rosti))",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76505",
-    "pr_number": 76505,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "76551": {
-    "commit": "7929c15ec7a9cb27272b7ecb098f13ef135ebde4",
-    "text": "proxy/transport: Support Content-Encoding: deflate",
-    "markdown": "proxy/transport: Support Content-Encoding: deflate ([#76551](https://github.com/kubernetes/kubernetes/pull/76551), [@JieJhih](https://github.com/JieJhih)) Courtesy of SIG API Machinery",
-    "author": "JieJhih",
-    "author_url": "https://github.com/JieJhih",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76551",
-    "pr_number": 76551,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "76573": {
-    "commit": "64a0441c2b43a13da0c3e614706e99c52392c449",
-    "text": "fix detach azure disk back off issue which has too big lock in failure retry condition",
-    "markdown": "fix detach azure disk back off issue which has too big lock in failure retry condition ([#76573](https://github.com/kubernetes/kubernetes/pull/76573), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76573",
-    "pr_number": 76573,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["azure", "cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76576": {
-    "commit": "b3847120241251d624d570e2212937c659614ea7",
-    "text": "Update to use go 1.12.4",
-    "markdown": "Update to use go 1.12.4 ([#76576](https://github.com/kubernetes/kubernetes/pull/76576), [@cblecker](https://github.com/cblecker)) Courtesy of SIG Release, and SIG Testing",
-    "author": "cblecker",
-    "author_url": "https://github.com/cblecker",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76576",
-    "pr_number": 76576,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["release", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "76595": {
-    "commit": "f44d04c1a205a4d650ec7ba3ec9a3a0ae0902366",
-    "text": "Remove the function Parallelize, please convert to use the function ParallelizeUntil.",
-    "markdown": "Remove the function Parallelize, please convert to use the function ParallelizeUntil. ([#76595](https://github.com/kubernetes/kubernetes/pull/76595), [@danielqsj](https://github.com/danielqsj))",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76595",
-    "pr_number": 76595,
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76603": {
-    "commit": "60a4c3c0030a5217a597902f05f587d47cd6eff8",
-    "text": "Fix TestEventChannelFull random fail",
-    "markdown": "Fix TestEventChannelFull random fail ([#76603](https://github.com/kubernetes/kubernetes/pull/76603), [@changyaowei](https://github.com/changyaowei))",
-    "author": "changyaowei",
-    "author_url": "https://github.com/changyaowei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76603",
-    "pr_number": 76603,
-    "areas": ["kubelet"],
-    "kinds": ["flake"],
-    "sigs": ["node"],
-    "release_version": "1.15.0"
-  },
-  "76627": {
-    "commit": "e1fa195451cf7d21e6ef080fe5bd046fb8d4fd4f",
-    "text": "GCE-only flag `cloud-provider-gce-lb-src-cidrs` becomes optional for external cloud providers.",
-    "markdown": "GCE-only flag `cloud-provider-gce-lb-src-cidrs` becomes optional for external cloud providers. ([#76627](https://github.com/kubernetes/kubernetes/pull/76627), [@timoreimann](https://github.com/timoreimann))",
-    "author": "timoreimann",
-    "author_url": "https://github.com/timoreimann",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76627",
-    "pr_number": 76627,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76628": {
-    "commit": "02ffdd1d5bbfdd7d6e129caaf3a5d71550817a61",
-    "text": "Juju provider source moved to the Charmed Kubernetes org",
-    "markdown": "Juju provider source moved to the Charmed Kubernetes org ([#76628](https://github.com/kubernetes/kubernetes/pull/76628), [@kwmonroe](https://github.com/kwmonroe))",
-    "author": "kwmonroe",
-    "author_url": "https://github.com/kwmonroe",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76628",
-    "pr_number": 76628,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "76636": {
-    "commit": "9746a40e44a3b208b67e30148e52f3dc6f7cfe20",
-    "text": "Kubeadm will now include the missing certificate key if it is unable to find an expected key during `kubeadm join` when used with the `--experimental-control-plane` flow",
-    "markdown": "Kubeadm will now include the missing certificate key if it is unable to find an expected key during `kubeadm join` when used with the `--experimental-control-plane` flow ([#76636](https://github.com/kubernetes/kubernetes/pull/76636), [@mdaniel](https://github.com/mdaniel))",
-    "author": "mdaniel",
-    "author_url": "https://github.com/mdaniel",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76636",
-    "pr_number": 76636,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "76640": {
-    "commit": "a961c13db53b4c28770b31331b5e581864dafcd2",
-    "text": "k8s-dns-node-cache image version v1.15.1",
-    "markdown": "k8s-dns-node-cache image version v1.15.1 ([#76640](https://github.com/kubernetes/kubernetes/pull/76640), [@george-angel](https://github.com/george-angel))",
-    "author": "george-angel",
-    "author_url": "https://github.com/george-angel",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76640",
-    "pr_number": 76640,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "76646": {
-    "commit": "e6188f8c295202f9dc35f30b65a4ac6d2180c3df",
-    "text": "Support print volumeMode using `kubectl get pv/pvc -o wide`",
-    "markdown": "Support print volumeMode using `kubectl get pv/pvc -o wide` ([#76646](https://github.com/kubernetes/kubernetes/pull/76646), [@cwdsuzhou](https://github.com/cwdsuzhou)) Courtesy of SIG CLI",
-    "author": "cwdsuzhou",
-    "author_url": "https://github.com/cwdsuzhou",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76646",
-    "pr_number": 76646,
-    "kinds": ["feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "76656": {
-    "commit": "f0a4672e6db7196312ef85731c19eadcbee1da9d",
-    "text": "Switch to instance-level update APIs for Azure VMSS loadbalancer operations",
-    "markdown": "Switch to instance-level update APIs for Azure VMSS loadbalancer operations ([#76656](https://github.com/kubernetes/kubernetes/pull/76656), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76656",
-    "pr_number": 76656,
-    "areas": ["cloudprovider", "dependency"],
-    "kinds": ["bug"],
-    "sigs": ["azure", "cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76659": {
-    "commit": "21bec91e895a828ee8ed8d45a7d58279b8cae46f",
-    "text": "Signal handling is initialized within hyperkube commands that require it (apiserver, kubelet)",
-    "markdown": "Signal handling is initialized within hyperkube commands that require it (apiserver, kubelet) ([#76659](https://github.com/kubernetes/kubernetes/pull/76659), [@S-Chan](https://github.com/S-Chan))",
-    "author": "S-Chan",
-    "author_url": "https://github.com/S-Chan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76659",
-    "pr_number": 76659,
-    "areas": ["apiserver", "kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "node", "release"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76675": {
-    "commit": "5ccda18fae27c474ec4e84537a8fe6cfa9d5dd96",
-    "text": "Fixes an error with stuck informers when an etcd watch receives update or delete events with missing data",
-    "markdown": "Fixes an error with stuck informers when an etcd watch receives update or delete events with missing data ([#76675](https://github.com/kubernetes/kubernetes/pull/76675), [@ryanmcnamara](https://github.com/ryanmcnamara))",
-    "author": "ryanmcnamara",
-    "author_url": "https://github.com/ryanmcnamara",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76675",
-    "pr_number": 76675,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "76691": {
-    "commit": "8f09fd6b2ab2bd52c000e8de06cb90f5c2bdc727",
-    "text": "Ensure the backend pools are set correctly for Azure SLB with multiple backend pools (e.g. outbound rules)",
-    "markdown": "Ensure the backend pools are set correctly for Azure SLB with multiple backend pools (e.g. outbound rules) ([#76691](https://github.com/kubernetes/kubernetes/pull/76691), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76691",
-    "pr_number": 76691,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["azure", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76710": {
-    "commit": "332d62a94a2bc8a7d0c69ca85fcd5c458816d677",
-    "text": "Introduce the v1beta2 config format to kubeadm.",
-    "markdown": "Introduce the v1beta2 config format to kubeadm. ([#76710](https://github.com/kubernetes/kubernetes/pull/76710), [@rosti](https://github.com/rosti))",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76710",
-    "pr_number": 76710,
-    "areas": ["kubeadm"],
-    "kinds": ["api-change"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "76711": {
-    "commit": "c58864de3f5f990b7dbcd25fcc93214eee03d127",
-    "text": "kube-up.sh no longer supports \"centos\" and \"local\" providers",
-    "markdown": "kube-up.sh no longer supports \"centos\" and \"local\" providers ([#76711](https://github.com/kubernetes/kubernetes/pull/76711), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76711",
-    "pr_number": 76711,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76713": {
-    "commit": "48f2be9cff605b79069fc7aa5795bebc2d83c495",
-    "text": "Remove deprecated --pod/-p flag from kubectl exec. The flag has been marked as deprecated since k8s version v1.12",
-    "markdown": "Remove deprecated --pod/-p flag from kubectl exec. The flag has been marked as deprecated since k8s version v1.12 ([#76713](https://github.com/kubernetes/kubernetes/pull/76713), [@prksu](https://github.com/prksu))",
-    "author": "prksu",
-    "author_url": "https://github.com/prksu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76713",
-    "pr_number": 76713,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.15.0"
-  },
-  "76722": {
-    "commit": "257e6863b8c33a24453fac540df068d1a6657cd4",
-    "text": "Windows nodes on GCE use a known-working 1809 image rather than the latest 1809 image.",
-    "markdown": "Windows nodes on GCE use a known-working 1809 image rather than the latest 1809 image. ([#76722](https://github.com/kubernetes/kubernetes/pull/76722), [@pjh](https://github.com/pjh))",
-    "author": "pjh",
-    "author_url": "https://github.com/pjh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76722",
-    "pr_number": 76722,
-    "kinds": ["bug", "failing-test"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76732": {
-    "commit": "a0b8d1cbb883483a8fc77fc4a1cc830fff290b32",
-    "text": "kube-proxy: os exit when CleanupAndExit is set to true",
-    "markdown": "kube-proxy: os exit when CleanupAndExit is set to true ([#76732](https://github.com/kubernetes/kubernetes/pull/76732), [@JieJhih](https://github.com/JieJhih))",
-    "author": "JieJhih",
-    "author_url": "https://github.com/JieJhih",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76732",
-    "pr_number": 76732,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.15.0"
-  },
-  "76749": {
-    "commit": "f5d958af4bbb38c562f4f11df6103790685abcc8",
-    "text": "Limit use of tags when calling EC2 API to prevent API throttling for very large clusters",
-    "markdown": "Limit use of tags when calling EC2 API to prevent API throttling for very large clusters ([#76749](https://github.com/kubernetes/kubernetes/pull/76749), [@mcrute](https://github.com/mcrute))",
-    "author": "mcrute",
-    "author_url": "https://github.com/mcrute",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76749",
-    "pr_number": 76749,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["aws", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76762": {
-    "commit": "d2ce69d9ad3d28c554c9a9560c15947181029ca8",
-    "text": "[fluentd-gcp addon] Bump fluentd-gcp-scaler to v0.5.2 to pick up security fixes.",
-    "markdown": "[fluentd-gcp addon] Bump fluentd-gcp-scaler to v0.5.2 to pick up security fixes. ([#76762](https://github.com/kubernetes/kubernetes/pull/76762), [@serathius](https://github.com/serathius))",
-    "author": "serathius",
-    "author_url": "https://github.com/serathius",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76762",
-    "pr_number": 76762,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "gcp"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76788": {
-    "commit": "b95fca093d29a0c63558138e1bfa3b434168e994",
-    "text": "Clean links handling in cp's tar code",
-    "markdown": "Clean links handling in cp's tar code ([#76788](https://github.com/kubernetes/kubernetes/pull/76788), [@soltysh](https://github.com/soltysh))",
-    "author": "soltysh",
-    "author_url": "https://github.com/soltysh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76788",
-    "pr_number": 76788,
-    "areas": ["kubectl", "security"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.15.0"
-  },
-  "76819": {
-    "commit": "b32b742d978ebdb195936ddfe28fd6efcf2a454b",
-    "text": "[metrics-server addon] Restore connecting to nodes via IP addresses",
-    "markdown": "[metrics-server addon] Restore connecting to nodes via IP addresses ([#76819](https://github.com/kubernetes/kubernetes/pull/76819), [@serathius](https://github.com/serathius))",
-    "author": "serathius",
-    "author_url": "https://github.com/serathius",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76819",
-    "pr_number": 76819,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76821": {
-    "commit": "36ccff1b27b6aee5c5599334842cb2b8cf8063f3",
-    "text": "kubeadm: Improve resiliency when it comes to updating the `kubeadm-config` config map upon new control plane joins or resets. This allows for safe multiple control plane joins and/or resets.",
-    "markdown": "kubeadm: Improve resiliency when it comes to updating the `kubeadm-config` config map upon new control plane joins or resets. This allows for safe multiple control plane joins and/or resets. ([#76821](https://github.com/kubernetes/kubernetes/pull/76821), [@ereslibre](https://github.com/ereslibre))",
-    "author": "ereslibre",
-    "author_url": "https://github.com/ereslibre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76821",
-    "pr_number": 76821,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "76848": {
-    "commit": "ccf9556ddcfdad779a5e072bb8415bbe9f559381",
-    "text": "Fixed a kubemark panic when hollow-node is morphed as proxy.",
-    "markdown": "Fixed a kubemark panic when hollow-node is morphed as proxy. ([#76848](https://github.com/kubernetes/kubernetes/pull/76848), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76848",
-    "pr_number": 76848,
-    "kinds": ["bug"],
-    "sigs": ["scalability"],
-    "release_version": "1.15.0"
-  },
-  "76849": {
-    "commit": "716344fd7d68f92d9a16280be1feae0ffe122bb4",
-    "text": "Admission webhooks are now properly called for `scale` and `deployments/rollback` subresources",
-    "markdown": "Admission webhooks are now properly called for `scale` and `deployments/rollback` subresources ([#76849](https://github.com/kubernetes/kubernetes/pull/76849), [@liggitt](https://github.com/liggitt)) Courtesy of SIG API Machinery, SIG Auth, and SIG Testing",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76849",
-    "pr_number": 76849,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "auth", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "76850": {
-    "commit": "aa11d0edef0d73afaf1c54f8ec37c6a11a1e4891",
-    "text": "GCE/Windows: add support for stackdriver logging agent",
-    "markdown": "GCE/Windows: add support for stackdriver logging agent ([#76850](https://github.com/kubernetes/kubernetes/pull/76850), [@yujuhong](https://github.com/yujuhong))",
-    "author": "yujuhong",
-    "author_url": "https://github.com/yujuhong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76850",
-    "pr_number": 76850,
-    "kinds": ["bug"],
-    "sigs": ["gcp"],
-    "release_version": "1.15.0"
-  },
-  "76862": {
-    "commit": "c88b7cdd58cb0efc79a9c1009eb5f927c3fc2b7e",
-    "text": "kubeadm: kubeadm upgrade now renews all the certificates used by one component before upgrading the component itself, with the exception of certificates signed by external CAs. User can eventually opt-out from certificate renewal during upgrades by setting the new flag --certificate-renewal to false.",
-    "markdown": "kubeadm: kubeadm upgrade now renews all the certificates used by one component before upgrading the component itself, with the exception of certificates signed by external CAs. User can eventually opt-out from certificate renewal during upgrades by setting the new flag --certificate-renewal to false. ([#76862](https://github.com/kubernetes/kubernetes/pull/76862), [@fabriziopandini](https://github.com/fabriziopandini))",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76862",
-    "pr_number": 76862,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "76865": {
-    "commit": "c262f9c7d9c4d6468f4dc28b2e61976dd6ff6115",
-    "text": "kubeadm: kubeadm alpha cert renew now ignores certificates signed by external CAs",
-    "markdown": "kubeadm: kubeadm alpha cert renew now ignores certificates signed by external CAs ([#76865](https://github.com/kubernetes/kubernetes/pull/76865), [@fabriziopandini](https://github.com/fabriziopandini))",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76865",
-    "pr_number": 76865,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "76909": {
-    "commit": "9b437f95207c04bf2f25ef3110fac9b356d1fa91",
-    "text": "Kubemark scripts have been fixed for IKS clusters.",
-    "markdown": "Kubemark scripts have been fixed for IKS clusters. ([#76909](https://github.com/kubernetes/kubernetes/pull/76909), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76909",
-    "pr_number": 76909,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "ibmcloud", "scalability", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76910": {
-    "commit": "90fbbee12950f336db2da94dda7beb87846f94e0",
-    "text": "Validating admission webhooks are now properly called for CREATE operations on the following resources: pods/binding, pods/eviction, bindings",
-    "markdown": "Validating admission webhooks are now properly called for CREATE operations on the following resources: pods/binding, pods/eviction, bindings ([#76910](https://github.com/kubernetes/kubernetes/pull/76910), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76910",
-    "pr_number": 76910,
-    "areas": ["admission-control", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["testing"],
-    "release_version": "1.15.0"
-  },
-  "76913": {
-    "commit": "82bfa667ed543b3fdf79f51a22389e2ba594da33",
-    "text": "This change enables a user to specify a DataSource/Kind of type \"PersistentVolumeClaim\" in their PVC spec.  This can then be detected by the external csi-provisioner and plugins if capable.",
-    "markdown": "This change enables a user to specify a DataSource/Kind of type \"PersistentVolumeClaim\" in their PVC spec.  This can then be detected by the external csi-provisioner and plugins if capable. ([#76913](https://github.com/kubernetes/kubernetes/pull/76913), [@j-griffith](https://github.com/j-griffith))",
-    "author": "j-griffith",
-    "author_url": "https://github.com/j-griffith",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76913",
-    "pr_number": 76913,
-    "kinds": ["api-change"],
-    "sigs": ["storage"],
-    "release_version": "1.15.0"
-  },
-  "76914": {
-    "commit": "219b166cb172344ee666a107161c07ba3805bd67",
-    "text": "Fix OpenID Connect (OIDC) token refresh when the client secret contains a special character.",
-    "markdown": "Fix OpenID Connect (OIDC) token refresh when the client secret contains a special character. ([#76914](https://github.com/kubernetes/kubernetes/pull/76914), [@tsuna](https://github.com/tsuna))",
-    "author": "tsuna",
-    "author_url": "https://github.com/tsuna",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76914",
-    "pr_number": 76914,
-    "areas": ["apiserver", "cloudprovider", "dependency"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth", "cli", "cloud-provider", "cluster-lifecycle", "node"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76915": {
-    "commit": "3aecf4350e3b518e8786df7422b9036f57832eb5",
-    "text": "Inline CSI ephemeral volumes can now be controlled with PodSecurityPolicy when the CSIInlineVolume alpha feature is enabled",
-    "markdown": "Inline CSI ephemeral volumes can now be controlled with PodSecurityPolicy when the CSIInlineVolume alpha feature is enabled ([#76915](https://github.com/kubernetes/kubernetes/pull/76915), [@vladimirvivien](https://github.com/vladimirvivien))",
-    "author": "vladimirvivien",
-    "author_url": "https://github.com/vladimirvivien",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76915",
-    "pr_number": 76915,
-    "areas": ["kubectl"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["auth", "cli", "storage"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76953": {
-    "commit": "0ff81d04dc5ff7870f8b0ced66a67b8c68eb7065",
-    "text": "Remove hyperkube short aliases from source code, Because hyperkube docker image currently create these aliases. If you are currently using /hyperkube {apiserver,scheduler,controller-manager,proxy} please switch to using the correct binary names.",
-    "markdown": "Remove hyperkube short aliases from source code, Because hyperkube docker image currently create these aliases. If you are currently using /hyperkube {apiserver,scheduler,controller-manager,proxy} please switch to using the correct binary names. ([#76953](https://github.com/kubernetes/kubernetes/pull/76953), [@Rand01ph](https://github.com/Rand01ph)) Courtesy of SIG Release",
-    "author": "Rand01ph",
-    "author_url": "https://github.com/Rand01ph",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76953",
-    "pr_number": 76953,
-    "kinds": ["cleanup"],
-    "sigs": ["release"],
-    "action_required": true,
-    "release_version": "1.15.0"
-  },
-  "76959": {
-    "commit": "279d4e1ca3b550ef7d193e93854e6eb10b937661",
-    "text": "Validating admission webhooks are now properly called for CREATE operations on the following resources: tokenreviews, subjectaccessreviews, localsubjectaccessreviews, selfsubjectaccessreviews, selfsubjectrulesreviews",
-    "markdown": "Validating admission webhooks are now properly called for CREATE operations on the following resources: tokenreviews, subjectaccessreviews, localsubjectaccessreviews, selfsubjectaccessreviews, selfsubjectrulesreviews ([#76959](https://github.com/kubernetes/kubernetes/pull/76959), [@sbezverk](https://github.com/sbezverk))",
-    "author": "sbezverk",
-    "author_url": "https://github.com/sbezverk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76959",
-    "pr_number": 76959,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["auth", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "76969": {
-    "commit": "28172ec17c088527bf543e65d2a2019ba7add20a",
-    "text": "Fixes a bug where dry-run is not honored for pod/eviction sub-resource.",
-    "markdown": "Fixes a bug where dry-run is not honored for pod/eviction sub-resource. ([#76969](https://github.com/kubernetes/kubernetes/pull/76969), [@apelisse](https://github.com/apelisse))",
-    "author": "apelisse",
-    "author_url": "https://github.com/apelisse",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76969",
-    "pr_number": 76969,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "76976": {
-    "commit": "805b6cbf83c8a60db735dd94e222432d451f634a",
-    "text": "`aws-cloud-provider` service account in the `kube-system` namespace need to be granted with list node permission with this optimization",
-    "markdown": "`aws-cloud-provider` service account in the `kube-system` namespace need to be granted with list node permission with this optimization ([#76976](https://github.com/kubernetes/kubernetes/pull/76976), [@zhan849](https://github.com/zhan849))",
-    "author": "zhan849",
-    "author_url": "https://github.com/zhan849",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76976",
-    "pr_number": 76976,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.0"
-  },
-  "76988": {
-    "commit": "2e96bf597eaa4e74650833916d6d984e93b77654",
-    "text": "---\nkind: StorageClass\napiVersion: storage.k8s.io/v1\nmetadata:\n  name: azurefile\nprovisioner: kubernetes.io/azure-file\nparameters:\n  skuName: Standard_LRS\n  resourceGroup: EXISTING_RESOURCE_GROUP_NAME\n  storageAccount: EXISTING_STORAGE_ACCOUNT_NAME\n  shareName: SHARE_NAME\n```\n\n**Which issue(s) this PR fixes**:\n\u003c!--\n*Automatically closes linked issue when PR is merged.\nUsage: `Fixes \u0026#35;\u003cissue number\u003e`, or `Fixes (paste link of issue)`.\n_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_--\u003e\nFixes \u0026#35;76986\n\n**Special notes for your reviewer**:\nadd shareName param in azure file storage class\n\n**Release note**:\n\u003c!--\nIf no, just write \"NONE\" in the release-note block below.\nIf yes, a release note is required:\nEnter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string \"action required\".\n--\u003e\n```\nspecify azure file share name in azure file plugin",
-    "markdown": "---\nkind: StorageClass\napiVersion: storage.k8s.io/v1\nmetadata:\n  name: azurefile\nprovisioner: kubernetes.io/azure-file\nparameters:\n  skuName: Standard_LRS\n  resourceGroup: EXISTING_RESOURCE_GROUP_NAME\n  storageAccount: EXISTING_STORAGE_ACCOUNT_NAME\n  shareName: SHARE_NAME\n```\n\n**Which issue(s) this PR fixes**:\n\u003c!--\n*Automatically closes linked issue when PR is merged.\nUsage: `Fixes \u0026#35;\u003cissue number\u003e`, or `Fixes (paste link of issue)`.\n_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_--\u003e\nFixes \u0026#35;76986\n\n**Special notes for your reviewer**:\nadd shareName param in azure file storage class\n\n**Release note**:\n\u003c!--\nIf no, just write \"NONE\" in the release-note block below.\nIf yes, a release note is required:\nEnter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string \"action required\".\n--\u003e\n```\nspecify azure file share name in azure file plugin ([#76988](https://github.com/kubernetes/kubernetes/pull/76988), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76988",
-    "pr_number": 76988,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["azure", "cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77010": {
-    "commit": "36e06bbb1fd3fc430ff4f8990684bbd53cf80e79",
-    "text": "Report cp errors consistently ",
-    "markdown": "Report cp errors consistently  ([#77010](https://github.com/kubernetes/kubernetes/pull/77010), [@soltysh](https://github.com/soltysh))",
-    "author": "soltysh",
-    "author_url": "https://github.com/soltysh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77010",
-    "pr_number": 77010,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.15.0"
-  },
-  "77012": {
-    "commit": "f29138c372ad7d81883d57ddf89830f5165bcb58",
-    "text": "kubeadm: Add ability to specify certificate encryption and decryption key for the upload/download certificates phases as part of the new v1beta2 kubeadm config format.",
-    "markdown": "kubeadm: Add ability to specify certificate encryption and decryption key for the upload/download certificates phases as part of the new v1beta2 kubeadm config format. ([#77012](https://github.com/kubernetes/kubernetes/pull/77012), [@rosti](https://github.com/rosti)) Courtesy of SIG Cluster Lifecycle",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77012",
-    "pr_number": 77012,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77022": {
-    "commit": "aefa6d4492dd014c27cfc6f08ee5fe94384f87a3",
-    "text": "API requests rejected by admission webhooks which specify an http status code \u003c 400 are now assigned a 400 status code.",
-    "markdown": "API requests rejected by admission webhooks which specify an http status code \u003c 400 are now assigned a 400 status code. ([#77022](https://github.com/kubernetes/kubernetes/pull/77022), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77022",
-    "pr_number": 77022,
-    "areas": ["admission-control", "apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "77051": {
-    "commit": "59a86822cb966ffb77a702321c89bb4e6f1c1c89",
-    "text": "kubeadm: bump the minimum supported Docker version to 1.13.1",
-    "markdown": "kubeadm: bump the minimum supported Docker version to 1.13.1 ([#77051](https://github.com/kubernetes/kubernetes/pull/77051), [@chenzhiwei](https://github.com/chenzhiwei))",
-    "author": "chenzhiwei",
-    "author_url": "https://github.com/chenzhiwei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77051",
-    "pr_number": 77051,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "77070": {
-    "commit": "9e29c3e39f916fe67654c9e06ada23e42217e532",
-    "text": "Upgrade go-autorest to v11.1.2",
-    "markdown": "Upgrade go-autorest to v11.1.2 ([#77070](https://github.com/kubernetes/kubernetes/pull/77070), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77070",
-    "pr_number": 77070,
-    "areas": ["apiserver", "cloudprovider", "dependency"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "azure", "cli", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77099": {
-    "commit": "2776cc1e726fb9fa74d3deddcd302b17a4cfb278",
-    "text": "Rancher credential provider has now been removed",
-    "markdown": "Rancher credential provider has now been removed ([#77099](https://github.com/kubernetes/kubernetes/pull/77099), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77099",
-    "pr_number": 77099,
-    "areas": ["dependency", "kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["cloud-provider", "node"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77136": {
-    "commit": "ba28e0e0fde4072aecb94e7d596e97d1c976a49e",
-    "text": "Promote meta.k8s.io/v1beta1 Table and PartialObjectMetadata to v1.",
-    "markdown": "Promote meta.k8s.io/v1beta1 Table and PartialObjectMetadata to v1. ([#77136](https://github.com/kubernetes/kubernetes/pull/77136), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77136",
-    "pr_number": 77136,
-    "areas": ["dependency"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "77139": {
-    "commit": "83f769c2a3111086af99a079ef276aefeb0366de",
-    "text": "ingress objects are now persisted in etcd using the networking.k8s.io/v1beta1 version",
-    "markdown": "ingress objects are now persisted in etcd using the networking.k8s.io/v1beta1 version ([#77139](https://github.com/kubernetes/kubernetes/pull/77139), [@cmluciano](https://github.com/cmluciano))",
-    "author": "cmluciano",
-    "author_url": "https://github.com/cmluciano",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77139",
-    "pr_number": 77139,
-    "areas": ["apiserver", "test"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "network", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77144": {
-    "commit": "167bc954fdb1a1d592b343de7a05fa57f58a5458",
-    "text": "When specifying an invalid value for a label, it was not always\nclear which label the value was specified for. Starting with this release, the\nlabel's key is included in such error messages, which makes debugging easier.",
-    "markdown": "When specifying an invalid value for a label, it was not always\nclear which label the value was specified for. Starting with this release, the\nlabel's key is included in such error messages, which makes debugging easier. ([#77144](https://github.com/kubernetes/kubernetes/pull/77144), [@kenegozi](https://github.com/kenegozi))",
-    "author": "kenegozi",
-    "author_url": "https://github.com/kenegozi",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77144",
-    "pr_number": 77144,
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "77170": {
-    "commit": "4891eaa3adbfafb61d2bd264d2f0daef124ee8b9",
-    "text": "DelayingQueue.ShutDown() is now able to be invoked multiple times without causing a closed channel panic.",
-    "markdown": "DelayingQueue.ShutDown() is now able to be invoked multiple times without causing a closed channel panic. ([#77170](https://github.com/kubernetes/kubernetes/pull/77170), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77170",
-    "pr_number": 77170,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "77180": {
-    "commit": "f9c9ecdbb62e5510dbdadf7ef4d3d48956562af7",
-    "text": "kubeadm: kubeadm alpha certs renew and kubeadm upgrade now supports renews of certificates embedded in KubeConfig files managed by kubeadm; this does not apply to certificates signed by external CAs. ",
-    "markdown": "kubeadm: kubeadm alpha certs renew and kubeadm upgrade now supports renews of certificates embedded in KubeConfig files managed by kubeadm; this does not apply to certificates signed by external CAs.  ([#77180](https://github.com/kubernetes/kubernetes/pull/77180), [@fabriziopandini](https://github.com/fabriziopandini)) Courtesy of SIG Cluster Lifecycle",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77180",
-    "pr_number": 77180,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77187": {
-    "commit": "16193d83a4725242768b222c1f3f8346c8f9b6ae",
-    "text": "fix azure disk list corruption issue",
-    "markdown": "fix azure disk list corruption issue ([#77187](https://github.com/kubernetes/kubernetes/pull/77187), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77187",
-    "pr_number": 77187,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["azure", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77207": {
-    "commit": "8efea56ac8e07c57ba93d5e6764e09b46d19f87e",
-    "text": "Add a condition NonStructuralSchema to CustomResourceDefinition listing Structural Schema violations as defined in KEP https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190425-structural-openapi.md. CRD authors should update their validation schemas to be structural in order to participate in future CRD features.\nCRD validation schemas get support for `x-kubernetes-embedded-resource: true` to specify `RawExtensions` in OpenAPI and to enforce coercing and validation of TypeMeta and ObjectMeta.\nCRD validation schemas get support for `x-kubernetes-int-or-string: true` to specify `intstr.IntOrString` in OpenAPI.\nCRD validation schemas get support for `x-kubernetes-preserve-unknown-fields: true` to control pruning via OpenAPI for parts of CustomResources.",
-    "markdown": "Add a condition NonStructuralSchema to CustomResourceDefinition listing Structural Schema violations as defined in KEP https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190425-structural-openapi.md. CRD authors should update their validation schemas to be structural in order to participate in future CRD features.\nCRD validation schemas get support for `x-kubernetes-embedded-resource: true` to specify `RawExtensions` in OpenAPI and to enforce coercing and validation of TypeMeta and ObjectMeta.\nCRD validation schemas get support for `x-kubernetes-int-or-string: true` to specify `intstr.IntOrString` in OpenAPI.\nCRD validation schemas get support for `x-kubernetes-preserve-unknown-fields: true` to control pruning via OpenAPI for parts of CustomResources. ([#77207](https://github.com/kubernetes/kubernetes/pull/77207), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77207",
-    "pr_number": 77207,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "77208": {
-    "commit": "e871241268c4aed15cd7d55692289297513bec87",
-    "text": "enhance the daemonset sync logic in clock-skew scenario",
-    "markdown": "enhance the daemonset sync logic in clock-skew scenario ([#77208](https://github.com/kubernetes/kubernetes/pull/77208), [@DaiHao](https://github.com/DaiHao))",
-    "author": "DaiHao",
-    "author_url": "https://github.com/DaiHao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77208",
-    "pr_number": 77208,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.15.0"
-  },
-  "77210": {
-    "commit": "5f6d9b614ed9637cadb8a6497293454f2142b348",
-    "text": "Add etag for NSG updates so as to fix nsg race condition",
-    "markdown": "Add etag for NSG updates so as to fix nsg race condition ([#77210](https://github.com/kubernetes/kubernetes/pull/77210), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77210",
-    "pr_number": 77210,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["azure", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77224": {
-    "commit": "62020ebe38ac5bbe72ca614974c6ebbe22a61986",
-    "text": "Fixes segmentation fault issue with Protobuf library when log entries are deeply nested.",
-    "markdown": "Fixes segmentation fault issue with Protobuf library when log entries are deeply nested. ([#77224](https://github.com/kubernetes/kubernetes/pull/77224), [@qingling128](https://github.com/qingling128))",
-    "author": "qingling128",
-    "author_url": "https://github.com/qingling128",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77224",
-    "pr_number": 77224,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "77245": {
-    "commit": "c4196408913d3ee2525d705d1636879b47b0c7cf",
-    "text": "fix issue that pull image failed from a cross-subscription Azure Container Registry when using MSI to authenticate",
-    "markdown": "fix issue that pull image failed from a cross-subscription Azure Container Registry when using MSI to authenticate ([#77245](https://github.com/kubernetes/kubernetes/pull/77245), [@norshtein](https://github.com/norshtein))",
-    "author": "norshtein",
-    "author_url": "https://github.com/norshtein",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77245",
-    "pr_number": 77245,
-    "kinds": ["bug"],
-    "sigs": ["azure"],
-    "release_version": "1.15.0"
-  },
-  "77267": {
-    "commit": "1e015237c702902133e655a8712b9004175745d0",
-    "text": "Preserves existing namespace information in manifests when running `kubectl set ... --local` commands",
-    "markdown": "Preserves existing namespace information in manifests when running `kubectl set ... --local` commands ([#77267](https://github.com/kubernetes/kubernetes/pull/77267), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77267",
-    "pr_number": 77267,
-    "areas": ["kubectl", "test"],
-    "kinds": ["bug"],
-    "sigs": ["cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77274": {
-    "commit": "fc005789c1162e55b22fb118dceaec5c0622b1db",
-    "text": "Enable 3rd party device monitoring by default",
-    "markdown": "Enable 3rd party device monitoring by default ([#77274](https://github.com/kubernetes/kubernetes/pull/77274), [@RenaudWasTaken](https://github.com/RenaudWasTaken)) Courtesy of SIG Node",
-    "author": "RenaudWasTaken",
-    "author_url": "https://github.com/RenaudWasTaken",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77274",
-    "pr_number": 77274,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77278": {
-    "commit": "005eb53ea1d9a0fac4f008f79a9ee559436def53",
-    "text": "API paging is now enabled by default in k8s.io/apiserver recommended options, and in k8s.io/sample-apiserver",
-    "markdown": "API paging is now enabled by default in k8s.io/apiserver recommended options, and in k8s.io/sample-apiserver ([#77278](https://github.com/kubernetes/kubernetes/pull/77278), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77278",
-    "pr_number": 77278,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "77282": {
-    "commit": "6027a38e78b4c257028d1a48fe3d90d8c02d99c4",
-    "text": "Bump addon-manager to v9.0.1\n- Rebase image on debian-base:v1.0.0",
-    "markdown": "Bump addon-manager to v9.0.1\n- Rebase image on debian-base:v1.0.0 ([#77282](https://github.com/kubernetes/kubernetes/pull/77282), [@MrHohn](https://github.com/MrHohn))",
-    "author": "MrHohn",
-    "author_url": "https://github.com/MrHohn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77282",
-    "pr_number": 77282,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77303": {
-    "commit": "53b41dd1fe1444b8c9bca948f5e469058474ff91",
-    "text": "Fixes spurious error messages about failing to clean up iptables rules when using iptables 1.8.",
-    "markdown": "Fixes spurious error messages about failing to clean up iptables rules when using iptables 1.8. ([#77303](https://github.com/kubernetes/kubernetes/pull/77303), [@danwinship](https://github.com/danwinship))",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77303",
-    "pr_number": 77303,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.15.0"
-  },
-  "77317": {
-    "commit": "9e52832c52c84c4b4b3dee6733ad392ff17e1753",
-    "text": "Fix bug with block volume expansion",
-    "markdown": "Fix bug with block volume expansion ([#77317](https://github.com/kubernetes/kubernetes/pull/77317), [@gnufied](https://github.com/gnufied))",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77317",
-    "pr_number": 77317,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77333": {
-    "commit": "3193e78a2f2d2296de0f16a1da3509b4b7ef7a49",
-    "text": "Add CRD spec.preserveUnknownFields boolean, defaulting to true in v1beta1 and to false in v1 CRDs. If false, fields not specified in the validation schema will be removed when sent to the API server or when read from etcd.",
-    "markdown": "Add CRD spec.preserveUnknownFields boolean, defaulting to true in v1beta1 and to false in v1 CRDs. If false, fields not specified in the validation schema will be removed when sent to the API server or when read from etcd. ([#77333](https://github.com/kubernetes/kubernetes/pull/77333), [@sttts](https://github.com/sttts)) Courtesy of SIG API Machinery, and SIG Testing",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77333",
-    "pr_number": 77333,
-    "areas": ["apiserver", "custom-resources", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77347": {
-    "commit": "df29573ec50375e831d833e27c63c23a88b983e8",
-    "text": "Fix kubectl describe CronJobs error of `Successful Job History Limit`.",
-    "markdown": "Fix kubectl describe CronJobs error of `Successful Job History Limit`. ([#77347](https://github.com/kubernetes/kubernetes/pull/77347), [@danielqsj](https://github.com/danielqsj))",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77347",
-    "pr_number": 77347,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.15.0"
-  },
-  "77368": {
-    "commit": "5bbad4fd4637e7bdaa3666e636539d9e2b9e031f",
-    "text": "Fixes incorrect handling by kubectl of custom resources whose Kind is \"Status\"",
-    "markdown": "Fixes incorrect handling by kubectl of custom resources whose Kind is \"Status\" ([#77368](https://github.com/kubernetes/kubernetes/pull/77368), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77368",
-    "pr_number": 77368,
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.15.0"
-  },
-  "77371": {
-    "commit": "aa25195ab1f046fcdf8886ef1b6cecdd8e1088bd",
-    "text": "Fix a bug where kube-proxy returns error due to existing ipset rules using a different hash type.",
-    "markdown": "Fix a bug where kube-proxy returns error due to existing ipset rules using a different hash type. ([#77371](https://github.com/kubernetes/kubernetes/pull/77371), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77371",
-    "pr_number": 77371,
-    "areas": ["ipvs"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.15.0"
-  },
-  "77378": {
-    "commit": "6a23a6278f48ef286b860159a9cab81d71e30640",
-    "text": "GCE/Windows: force kill Stackdriver logging processes when the service cannot be stopped",
-    "markdown": "GCE/Windows: force kill Stackdriver logging processes when the service cannot be stopped ([#77378](https://github.com/kubernetes/kubernetes/pull/77378), [@yujuhong](https://github.com/yujuhong))",
-    "author": "yujuhong",
-    "author_url": "https://github.com/yujuhong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77378",
-    "pr_number": 77378,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "gcp"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77412": {
-    "commit": "1dd550cf7c1b9ab54c931d4e0976197f31d1b304",
-    "text": "The GCERegionalPersistentDisk feature gate (GA in 1.13) can no longer be disabled. The feature gate will be removed in v1.17.",
-    "markdown": "The GCERegionalPersistentDisk feature gate (GA in 1.13) can no longer be disabled. The feature gate will be removed in v1.17. ([#77412](https://github.com/kubernetes/kubernetes/pull/77412), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77412",
-    "pr_number": 77412,
-    "areas": ["cloudprovider"],
-    "kinds": ["cleanup"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77415": {
-    "commit": "291f2f863ab6f73787962f65b0f38998f3c9ac00",
-    "text": "Allow to define kubeconfig file for OpenStack cloud provider.",
-    "markdown": "Allow to define kubeconfig file for OpenStack cloud provider. ([#77415](https://github.com/kubernetes/kubernetes/pull/77415), [@Fedosin](https://github.com/Fedosin)) Courtesy of SIG Cloud Provider, and SIG Storage",
-    "author": "Fedosin",
-    "author_url": "https://github.com/Fedosin",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77415",
-    "pr_number": 77415,
-    "areas": ["cloudprovider"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider", "storage"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77423": {
-    "commit": "6dfc9961567ae361461f7fc91083ff7d80b617c7",
-    "text": "`kubectl rollout restart` now works for daemonsets and statefulsets.",
-    "markdown": "`kubectl rollout restart` now works for daemonsets and statefulsets. ([#77423](https://github.com/kubernetes/kubernetes/pull/77423), [@apelisse](https://github.com/apelisse)) Courtesy of SIG CLI, and SIG Testing",
-    "author": "apelisse",
-    "author_url": "https://github.com/apelisse",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77423",
-    "pr_number": 77423,
-    "areas": ["kubectl", "test"],
-    "kinds": ["feature"],
-    "sigs": ["cli", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77426": {
-    "commit": "946087b422e38dba37a8418860b0f2ede8597394",
-    "text": "If a pod has a running instance, the stats of its previously terminated instances will not show up in the kubelet summary stats any more for CRI runtimes like containerd and cri-o.\n\nThis keeps the behavior consistent with Docker integration, and fixes an issue that some container Prometheus metrics don't work when there are summary stats for multiple instances of the same pod.",
-    "markdown": "If a pod has a running instance, the stats of its previously terminated instances will not show up in the kubelet summary stats any more for CRI runtimes like containerd and cri-o.\n\nThis keeps the behavior consistent with Docker integration, and fixes an issue that some container Prometheus metrics don't work when there are summary stats for multiple instances of the same pod. ([#77426](https://github.com/kubernetes/kubernetes/pull/77426), [@Random-Liu](https://github.com/Random-Liu))",
-    "author": "Random-Liu",
-    "author_url": "https://github.com/Random-Liu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77426",
-    "pr_number": 77426,
-    "areas": ["kubelet"],
-    "kinds": ["bug", "failing-test"],
-    "sigs": ["instrumentation", "node"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77447": {
-    "commit": "0befec7fb6e53335e6672c464ddb3fb4343bfef2",
-    "text": "API servers using the default Google Compute Engine bootstrapping scripts will have their insecure port (`:8080`) disabled by default. To enable the insecure port, set `ENABLE_APISERVER_INSECURE_PORT=true` in kube-env or as an environment variable.",
-    "markdown": "API servers using the default Google Compute Engine bootstrapping scripts will have their insecure port (`:8080`) disabled by default. To enable the insecure port, set `ENABLE_APISERVER_INSECURE_PORT=true` in kube-env or as an environment variable. ([#77447](https://github.com/kubernetes/kubernetes/pull/77447), [@dekkagaijin](https://github.com/dekkagaijin))",
-    "author": "dekkagaijin",
-    "author_url": "https://github.com/dekkagaijin",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77447",
-    "pr_number": 77447,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "gcp"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77448": {
-    "commit": "274876ef9af526575f480c977f7dd7c4bb26e760",
-    "text": "Clients may now request that API objects are converted to the `v1.Table` and `v1.PartialObjectMetadata` forms for generic access to objects.",
-    "markdown": "Clients may now request that API objects are converted to the `v1.Table` and `v1.PartialObjectMetadata` forms for generic access to objects. ([#77448](https://github.com/kubernetes/kubernetes/pull/77448), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77448",
-    "pr_number": 77448,
-    "areas": ["apiserver", "kubectl", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77457": {
-    "commit": "1ce103a41f98af6b7ec7f574fbbf7ef7356c5c82",
-    "text": "Implement un-reserve extension point for the scheduling framework.",
-    "markdown": "Implement un-reserve extension point for the scheduling framework. ([#77457](https://github.com/kubernetes/kubernetes/pull/77457), [@danielqsj](https://github.com/danielqsj)) Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77457",
-    "pr_number": 77457,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77458": {
-    "commit": "b34d7ac0ce0c43ee4bec1e15185b13e44a2f7843",
-    "text": "GCE clusters will include some IP ranges that are not in used on the public Internet to the list of non-masq IPs.\nBump ip-masq-agent version to v2.3.0 with flag `nomasq-all-reserved-ranges` turned on.",
-    "markdown": "GCE clusters will include some IP ranges that are not in used on the public Internet to the list of non-masq IPs.\nBump ip-masq-agent version to v2.3.0 with flag `nomasq-all-reserved-ranges` turned on. ([#77458](https://github.com/kubernetes/kubernetes/pull/77458), [@grayluck](https://github.com/grayluck)) Courtesy of SIG Cluster Lifecycle, and SIG Network",
-    "author": "grayluck",
-    "author_url": "https://github.com/grayluck",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77458",
-    "pr_number": 77458,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77483": {
-    "commit": "ed239cefa6b3add3d53d55e02fa9c28a0bbfc25a",
-    "text": "remove VM API call dep in azure disk WaitForAttach",
-    "markdown": "remove VM API call dep in azure disk WaitForAttach ([#77483](https://github.com/kubernetes/kubernetes/pull/77483), [@andyzhangx](https://github.com/andyzhangx)) Courtesy of SIG Azure, and SIG Storage",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77483",
-    "pr_number": 77483,
-    "kinds": ["bug", "feature"],
-    "sigs": ["azure", "storage"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77490": {
-    "commit": "b9bde60cfd01cf657fec0dadba826c57aec4dcf9",
-    "text": "Fix race conditions for Azure loadbalancer and route updates.",
-    "markdown": "Fix race conditions for Azure loadbalancer and route updates. ([#77490](https://github.com/kubernetes/kubernetes/pull/77490), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77490",
-    "pr_number": 77490,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["azure", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77491": {
-    "commit": "1c689c2f49b5e03d4b12c5a4247ed30ac1c1bddb",
-    "text": "add operation name for vm/vmss update operations in prometheus metrics",
-    "markdown": "add operation name for vm/vmss update operations in prometheus metrics ([#77491](https://github.com/kubernetes/kubernetes/pull/77491), [@andyzhangx](https://github.com/andyzhangx)) Courtesy of SIG Azure, and SIG Cloud Provider",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77491",
-    "pr_number": 77491,
-    "areas": ["cloudprovider"],
-    "kinds": ["feature"],
-    "sigs": ["azure", "cloud-provider"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77495": {
-    "commit": "8ea9edbb0290e9de1e6d274e816a4002892cca6f",
-    "text": "Client:\n Version:           19.03.0-beta3\n API version:       1.40\n Go version:        go1.12.4\n Git commit:        c55e026\n Built:             Thu Apr 25 03:03:18 2019\n OS/Arch:           linux/amd64\n Experimental:      false\n\nServer:\n Engine:\n  Version:          19.03.0-beta3\n  API version:      1.40 (minimum version 1.12)\n  Go version:       go1.12.4\n  Git commit:       c55e026\n  Built:            Thu Apr 25 03:01:56 2019\n  OS/Arch:          linux/amd64\n  Experimental:     false\n containerd:\n  Version:          1.2.5\n  GitCommit:        bb71b10fd8f58240ca47fbb579b9d1028eea7c84\n runc:\n  Version:          1.0.0-rc6+dev\n  GitCommit:        2b18fe1d885ee5083ef9f0838fee39b62d653e30\n docker-init:\n  Version:          0.18.0\n  GitCommit:        fec3683\n```\n**Which issue(s) this PR fixes**:\n\u003c!--\n*Automatically closes linked issue when PR is merged.\nUsage: `Fixes \u0026#35;\u003cissue number\u003e`, or `Fixes (paste link of issue)`.\n_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_--\u003e\nFixes \u0026#35;\n\n**Special notes for your reviewer**:\n\n**Does this PR introduce a user-facing change?**:\n\u003c!--\nIf no, just write \"NONE\" in the release-note block below.\nIf yes, a release note is required:\nEnter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string \"action required\".\n--\u003e\n```\nFix local-up-cluster.sh didn't work with docker 19.03.0-beta3 ",
-    "markdown": "Client:\n Version:           19.03.0-beta3\n API version:       1.40\n Go version:        go1.12.4\n Git commit:        c55e026\n Built:             Thu Apr 25 03:03:18 2019\n OS/Arch:           linux/amd64\n Experimental:      false\n\nServer:\n Engine:\n  Version:          19.03.0-beta3\n  API version:      1.40 (minimum version 1.12)\n  Go version:       go1.12.4\n  Git commit:       c55e026\n  Built:            Thu Apr 25 03:01:56 2019\n  OS/Arch:          linux/amd64\n  Experimental:     false\n containerd:\n  Version:          1.2.5\n  GitCommit:        bb71b10fd8f58240ca47fbb579b9d1028eea7c84\n runc:\n  Version:          1.0.0-rc6+dev\n  GitCommit:        2b18fe1d885ee5083ef9f0838fee39b62d653e30\n docker-init:\n  Version:          0.18.0\n  GitCommit:        fec3683\n```\n**Which issue(s) this PR fixes**:\n\u003c!--\n*Automatically closes linked issue when PR is merged.\nUsage: `Fixes \u0026#35;\u003cissue number\u003e`, or `Fixes (paste link of issue)`.\n_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_--\u003e\nFixes \u0026#35;\n\n**Special notes for your reviewer**:\n\n**Does this PR introduce a user-facing change?**:\n\u003c!--\nIf no, just write \"NONE\" in the release-note block below.\nIf yes, a release note is required:\nEnter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string \"action required\".\n--\u003e\n```\nFix local-up-cluster.sh didn't work with docker 19.03.0-beta3  ([#77495](https://github.com/kubernetes/kubernetes/pull/77495), [@wanghaoran1988](https://github.com/wanghaoran1988))",
-    "author": "wanghaoran1988",
-    "author_url": "https://github.com/wanghaoran1988",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77495",
-    "pr_number": 77495,
-    "kinds": ["bug"],
-    "sigs": ["testing"],
-    "release_version": "1.15.0"
-  },
-  "77501": {
-    "commit": "df4f033a4f0c5103cd703fedcc91b16bfdb33106",
-    "text": "Add configuration options for the scheduling framework and its plugins.",
-    "markdown": "Add configuration options for the scheduling framework and its plugins. ([#77501](https://github.com/kubernetes/kubernetes/pull/77501), [@JieJhih](https://github.com/JieJhih)) Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "JieJhih",
-    "author_url": "https://github.com/JieJhih",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77501",
-    "pr_number": 77501,
-    "areas": ["test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77507": {
-    "commit": "c854f72dcba5ca465d978e6e01ebffdaa4c9d2c4",
-    "text": "kubeadm: fix a bug related to volume unmount if the kubelet run directory is a symbolic link",
-    "markdown": "kubeadm: fix a bug related to volume unmount if the kubelet run directory is a symbolic link ([#77507](https://github.com/kubernetes/kubernetes/pull/77507), [@cuericlee](https://github.com/cuericlee))",
-    "author": "cuericlee",
-    "author_url": "https://github.com/cuericlee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77507",
-    "pr_number": 77507,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "77516": {
-    "commit": "3e7fa617b30963ba4fe8e183a1a2a640fb821b69",
-    "text": "Add a field to store CSI volume expansion secrets",
-    "markdown": "Add a field to store CSI volume expansion secrets ([#77516](https://github.com/kubernetes/kubernetes/pull/77516), [@gnufied](https://github.com/gnufied))",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77516",
-    "pr_number": 77516,
-    "kinds": ["api-change"],
-    "sigs": ["apps", "storage"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77523": {
-    "commit": "bdf3d248eb401f48bb0ed5a159cb4264803dd789",
-    "text": "iptables proxier: route local traffic to LB IPs to service chain",
-    "markdown": "iptables proxier: route local traffic to LB IPs to service chain ([#77523](https://github.com/kubernetes/kubernetes/pull/77523), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77523",
-    "pr_number": 77523,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.15.0"
-  },
-  "77528": {
-    "commit": "851afa0beaac20757ad2556af94e31dce0a918be",
-    "text": "Update to use go 1.12.5",
-    "markdown": "Update to use go 1.12.5 ([#77528](https://github.com/kubernetes/kubernetes/pull/77528), [@cblecker](https://github.com/cblecker))",
-    "author": "cblecker",
-    "author_url": "https://github.com/cblecker",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77528",
-    "pr_number": 77528,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["release", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77529": {
-    "commit": "796ecb939183ed70f4b8508cfc5bee3fba507b38",
-    "text": "Support \"queue-sort\" extension point for scheduling framework",
-    "markdown": "Support \"queue-sort\" extension point for scheduling framework ([#77529](https://github.com/kubernetes/kubernetes/pull/77529), [@draveness](https://github.com/draveness)) Courtesy of SIG Scheduling",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77529",
-    "pr_number": 77529,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77559": {
-    "commit": "c54b664b1c937555f80b5ed70092c1676a971d51",
-    "text": "Implement Permit extension point of the scheduling framework.",
-    "markdown": "Implement Permit extension point of the scheduling framework. ([#77559](https://github.com/kubernetes/kubernetes/pull/77559), [@ahg-g](https://github.com/ahg-g)) Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "ahg-g",
-    "author_url": "https://github.com/ahg-g",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77559",
-    "pr_number": 77559,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77563": {
-    "commit": "0b6ad8bc3f0e85e3a0f1a8a94da3ca4068506de3",
-    "text": "Add `Option` field to the admission webhook `AdmissionReview` API that provides the operation options (e.g. `DeleteOption` or `CreateOption`) for the operation being performed.",
-    "markdown": "Add `Option` field to the admission webhook `AdmissionReview` API that provides the operation options (e.g. `DeleteOption` or `CreateOption`) for the operation being performed. ([#77563](https://github.com/kubernetes/kubernetes/pull/77563), [@jpbetz](https://github.com/jpbetz)) Courtesy of SIG API Machinery, SIG Auth, and SIG Testing",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77563",
-    "pr_number": 77563,
-    "areas": ["apiserver", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "auth", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77567": {
-    "commit": "bb475c4a645afa96a648cd90342e70d2884206c3",
-    "text": "Add Post-bind extension point to the scheduling framework",
-    "markdown": "Add Post-bind extension point to the scheduling framework ([#77567](https://github.com/kubernetes/kubernetes/pull/77567), [@wgliang](https://github.com/wgliang)) Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77567",
-    "pr_number": 77567,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77589": {
-    "commit": "74469297fe75b3177aec099a2d794333d4e054c7",
-    "text": "Fix kubectl exec usage string",
-    "markdown": "Fix kubectl exec usage string ([#77589](https://github.com/kubernetes/kubernetes/pull/77589), [@soltysh](https://github.com/soltysh))",
-    "author": "soltysh",
-    "author_url": "https://github.com/soltysh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77589",
-    "pr_number": 77589,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.15.0"
-  },
-  "77598": {
-    "commit": "b9ccdd2824c8df7711d3aed985d3e1b549a40de3",
-    "text": "Add Un-reserve extension point for the scheduling framework.",
-    "markdown": "Add Un-reserve extension point for the scheduling framework. ([#77598](https://github.com/kubernetes/kubernetes/pull/77598), [@danielqsj](https://github.com/danielqsj)) Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77598",
-    "pr_number": 77598,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77611": {
-    "commit": "6e78282a822e5b4aaf974c3430ff578f0d7ababc",
-    "text": "kube-apiserver: the `--enable-logs-handler` flag and log-serving functionality is deprecated, and scheduled to be removed in v1.19.",
-    "markdown": "kube-apiserver: the `--enable-logs-handler` flag and log-serving functionality is deprecated, and scheduled to be removed in v1.19. ([#77611](https://github.com/kubernetes/kubernetes/pull/77611), [@rohitsardesai83](https://github.com/rohitsardesai83))",
-    "author": "rohitsardesai83",
-    "author_url": "https://github.com/rohitsardesai83",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77611",
-    "pr_number": 77611,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "77617": {
-    "commit": "33f44a29b829a8715cd6246935edfd82969e67f9",
-    "text": "ingress:  Update in-tree Ingress controllers, examples, and clients to target networking.k8s.io/v1beta1",
-    "markdown": "ingress:  Update in-tree Ingress controllers, examples, and clients to target networking.k8s.io/v1beta1 ([#77617](https://github.com/kubernetes/kubernetes/pull/77617), [@cmluciano](https://github.com/cmluciano))",
-    "author": "cmluciano",
-    "author_url": "https://github.com/cmluciano",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77617",
-    "pr_number": 77617,
-    "areas": ["e2e-test-framework", "kubectl", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli", "network", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77619": {
-    "commit": "ed4c50852452a30369bdf7004b1c06043caf8c92",
-    "text": "Fixed a bug in the apiserver storage that could cause just-added finalizers to be ignored on an immediately following delete request, leading to premature deletion.",
-    "markdown": "Fixed a bug in the apiserver storage that could cause just-added finalizers to be ignored on an immediately following delete request, leading to premature deletion. ([#77619](https://github.com/kubernetes/kubernetes/pull/77619), [@caesarxuchao](https://github.com/caesarxuchao))",
-    "author": "caesarxuchao",
-    "author_url": "https://github.com/caesarxuchao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77619",
-    "pr_number": 77619,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "77630": {
-    "commit": "e9af72c6e9524ed85c827298f87220e324444d36",
-    "text": "Fix public IPs issues when multiple clusters are sharing the same resource group.\n\n\n\nIf the cluster is upgraded from old releases and the same resource group would be shared by multiple clusters, please recreate those LoadBalancer services or add a new tag 'kubernetes-cluster-name: \u003ccluster-name\u003e' manually for existing public IPs.\nFor multiple clusters sharing the same resource group, they should be configured with different cluster name by `kube-controller-manager --cluster-name=\u003ccluster-name\u003e`",
-    "markdown": "Fix public IPs issues when multiple clusters are sharing the same resource group.\n\n\n\nIf the cluster is upgraded from old releases and the same resource group would be shared by multiple clusters, please recreate those LoadBalancer services or add a new tag 'kubernetes-cluster-name: \u003ccluster-name\u003e' manually for existing public IPs.\nFor multiple clusters sharing the same resource group, they should be configured with different cluster name by `kube-controller-manager --cluster-name=\u003ccluster-name\u003e` ([#77630](https://github.com/kubernetes/kubernetes/pull/77630), [@feiskyer](https://github.com/feiskyer)) Courtesy of SIG Azure, and SIG Cloud Provider",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77630",
-    "pr_number": 77630,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["azure", "cloud-provider"],
-    "action_required": true,
-    "release_version": "1.15.0"
-  },
-  "77653": {
-    "commit": "fb41b7a801538c6c8c9385dc35bf1eaae92f06ab",
-    "text": "CRD validation schemas should not specify `metadata` fields other than `name` and `generateName`. A schema will not be considered structural (and therefore ready for future features) if `metadata` is specified in any other way.",
-    "markdown": "CRD validation schemas should not specify `metadata` fields other than `name` and `generateName`. A schema will not be considered structural (and therefore ready for future features) if `metadata` is specified in any other way. ([#77653](https://github.com/kubernetes/kubernetes/pull/77653), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77653",
-    "pr_number": 77653,
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "77656": {
-    "commit": "ef9e794a369d25f4024317dc5ca5bc04c18f887e",
-    "text": "Check if container memory stats are available before accessing it",
-    "markdown": "Check if container memory stats are available before accessing it ([#77656](https://github.com/kubernetes/kubernetes/pull/77656), [@yastij](https://github.com/yastij))",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77656",
-    "pr_number": 77656,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.15.0"
-  },
-  "77719": {
-    "commit": "bf79c7c7eedf2e79a721259cf0cbf519cdea623c",
-    "text": "Fix some service tags not supported issues for Azure LoadBalancer service",
-    "markdown": "Fix some service tags not supported issues for Azure LoadBalancer service ([#77719](https://github.com/kubernetes/kubernetes/pull/77719), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77719",
-    "pr_number": 77719,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["azure", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77722": {
-    "commit": "3bc595e37c144882ed4c2a23b8bef4ba82cfffcd",
-    "text": "fix incorrect prometheus azure metrics",
-    "markdown": "fix incorrect prometheus azure metrics ([#77722](https://github.com/kubernetes/kubernetes/pull/77722), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77722",
-    "pr_number": 77722,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["azure", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77739": {
-    "commit": "6a0db7b1894fd7c993c761ccaea1d3984ed70f7f",
-    "text": "kubeadm: v1beta2 InitConfiguration no longer embeds ClusterConfiguration it it.",
-    "markdown": "kubeadm: v1beta2 InitConfiguration no longer embeds ClusterConfiguration it it. ([#77739](https://github.com/kubernetes/kubernetes/pull/77739), [@rosti](https://github.com/rosti))",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77739",
-    "pr_number": 77739,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "77743": {
-    "commit": "358bfe3808c1d3d13e63f56de22c31c8ae5e859d",
-    "text": "In CRD webhook conversion ignore changes to metadata other than for labels and annotations.",
-    "markdown": "In CRD webhook conversion ignore changes to metadata other than for labels and annotations. ([#77743](https://github.com/kubernetes/kubernetes/pull/77743), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77743",
-    "pr_number": 77743,
-    "areas": ["custom-resources", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77755": {
-    "commit": "99f2e78c9a5673bf0f5db1b636d6307352946f29",
-    "text": "Move online volume expansion to beta",
-    "markdown": "Move online volume expansion to beta ([#77755](https://github.com/kubernetes/kubernetes/pull/77755), [@gnufied](https://github.com/gnufied)) Courtesy of SIG Node, SIG Storage, and SIG Testing",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77755",
-    "pr_number": 77755,
-    "areas": ["kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["node", "storage", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77773": {
-    "commit": "6ba13bf151290ae3fd3c6acfd017e233f21ccd59",
-    "text": "Fix a bug that causes DaemonSet rolling update to hang when its pod gets stuck at terminating. ",
-    "markdown": "Fix a bug that causes DaemonSet rolling update to hang when its pod gets stuck at terminating.  ([#77773](https://github.com/kubernetes/kubernetes/pull/77773), [@DaiHao](https://github.com/DaiHao))",
-    "author": "DaiHao",
-    "author_url": "https://github.com/DaiHao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77773",
-    "pr_number": 77773,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.15.0"
-  },
-  "77780": {
-    "commit": "e1770e698e97437d0fdf0a8a0351406cf6c135d1",
-    "text": "kubeadm: kubeadm alpha certs renew  --csr-only now reads the current certificates as the authoritative source for certificates attributes (same as kubeadm alpha certs renew)",
-    "markdown": "kubeadm: kubeadm alpha certs renew  --csr-only now reads the current certificates as the authoritative source for certificates attributes (same as kubeadm alpha certs renew) ([#77780](https://github.com/kubernetes/kubernetes/pull/77780), [@fabriziopandini](https://github.com/fabriziopandini))",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77780",
-    "pr_number": 77780,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "77802": {
-    "commit": "944a7e2be685aade95202aba38b6683ba399963f",
-    "text": "IPVS: Disable graceful termination for UDP traffic to solve issues with high number of UDP connections (DNS / syslog in particular)",
-    "markdown": "IPVS: Disable graceful termination for UDP traffic to solve issues with high number of UDP connections (DNS / syslog in particular) ([#77802](https://github.com/kubernetes/kubernetes/pull/77802), [@lbernail](https://github.com/lbernail))",
-    "author": "lbernail",
-    "author_url": "https://github.com/lbernail",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77802",
-    "pr_number": 77802,
-    "areas": ["ipvs"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.15.0"
-  },
-  "77816": {
-    "commit": "c85c0e4780e428fdfb33debc31bc88f8095e5831",
-    "text": "Fixed a transient error API requests for custom resources could encounter while changes to the CustomResourceDefinition were being applied.",
-    "markdown": "Fixed a transient error API requests for custom resources could encounter while changes to the CustomResourceDefinition were being applied. ([#77816](https://github.com/kubernetes/kubernetes/pull/77816), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77816",
-    "pr_number": 77816,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "77820": {
-    "commit": "4f33b5f42146400c9aa0a1f0f5ad79607ec06f6d",
-    "text": "Deprecated Kubelet security controls AllowPrivileged, HostNetworkSources, HostPIDSources, HostIPCSources have been removed. Enforcement of these restrictions should be done through admission control instead (e.g. PodSecurityPolicy).\nThe deprecated Kubelet flag `--allow-privileged` has been removed. Remove any use of `--allow-privileged` from your kubelet scripts or manifests.",
-    "markdown": "Deprecated Kubelet security controls AllowPrivileged, HostNetworkSources, HostPIDSources, HostIPCSources have been removed. Enforcement of these restrictions should be done through admission control instead (e.g. PodSecurityPolicy).\nThe deprecated Kubelet flag `--allow-privileged` has been removed. Remove any use of `--allow-privileged` from your kubelet scripts or manifests. ([#77820](https://github.com/kubernetes/kubernetes/pull/77820), [@dims](https://github.com/dims)) Courtesy of SIG Cluster Lifecycle, SIG GCP, SIG Node, and SIG Testing",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77820",
-    "pr_number": 77820,
-    "areas": ["kubelet", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "gcp", "node", "testing"],
-    "action_required": true,
-    "release_version": "1.15.0"
-  },
-  "77825": {
-    "commit": "a07b027261383127494fc6d4e3ca2c544fb4a464",
-    "text": "The CustomResourcePublishOpenAPI feature is now beta and enabled by default. CustomResourceDefinitions with [structural schemas](https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190425-structural-openapi.md) now publish schemas in the OpenAPI document served at `/openapi/v2`. CustomResourceDefinitions with non-structural schemas have a `NonStructuralSchema` condition added with details about what needs to be corrected in the validation schema.",
-    "markdown": "The CustomResourcePublishOpenAPI feature is now beta and enabled by default. CustomResourceDefinitions with [structural schemas](https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190425-structural-openapi.md) now publish schemas in the OpenAPI document served at `/openapi/v2`. CustomResourceDefinitions with non-structural schemas have a `NonStructuralSchema` condition added with details about what needs to be corrected in the validation schema. ([#77825](https://github.com/kubernetes/kubernetes/pull/77825), [@roycaihw](https://github.com/roycaihw)) Courtesy of SIG API Machinery, and SIG Testing",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77825",
-    "pr_number": 77825,
-    "areas": ["custom-resources", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77843": {
-    "commit": "76c214b259e2542613e4183eb2c5fa02423c70e8",
-    "text": "Publish DeleteOptions parameters for deletecollection endpoints in OpenAPI spec",
-    "markdown": "Publish DeleteOptions parameters for deletecollection endpoints in OpenAPI spec ([#77843](https://github.com/kubernetes/kubernetes/pull/77843), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77843",
-    "pr_number": 77843,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "77847": {
-    "commit": "d823fa23c674363c40e9e196f24b88018f04614f",
-    "text": "kubeadm: expose the kubeadm reset command as phases",
-    "markdown": "kubeadm: expose the kubeadm reset command as phases ([#77847](https://github.com/kubernetes/kubernetes/pull/77847), [@yagonobre](https://github.com/yagonobre))",
-    "author": "yagonobre",
-    "author_url": "https://github.com/yagonobre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77847",
-    "pr_number": 77847,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "77848": {
-    "commit": "6118b8aa735b8ee7245fb851b8fe9199d778e30e",
-    "text": "Add `kubeadm alpha certs certificate-key` command to generate secure random key to use on `kubeadm init --experimental-upload-certs`",
-    "markdown": "Add `kubeadm alpha certs certificate-key` command to generate secure random key to use on `kubeadm init --experimental-upload-certs` ([#77848](https://github.com/kubernetes/kubernetes/pull/77848), [@yagonobre](https://github.com/yagonobre)) Courtesy of SIG Cluster Lifecycle",
-    "author": "yagonobre",
-    "author_url": "https://github.com/yagonobre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77848",
-    "pr_number": 77848,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77851": {
-    "commit": "b5336132336d9a8696001594b716c85084f98e02",
-    "text": "remove vmsizelist call in azure disk GetVolumeLimits which happens in kubelet finally",
-    "markdown": "remove vmsizelist call in azure disk GetVolumeLimits which happens in kubelet finally ([#77851](https://github.com/kubernetes/kubernetes/pull/77851), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77851",
-    "pr_number": 77851,
-    "kinds": ["bug"],
-    "sigs": ["azure", "storage"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77863": {
-    "commit": "81a61ae0e37143299ee5947a6c2c5195ec5f72ae",
-    "text": "Kubeadm: a new command `kubeadm alpha certs check-expiration` was created in order to help users in managing expiration for local PKI certificates",
-    "markdown": "Kubeadm: a new command `kubeadm alpha certs check-expiration` was created in order to help users in managing expiration for local PKI certificates ([#77863](https://github.com/kubernetes/kubernetes/pull/77863), [@fabriziopandini](https://github.com/fabriziopandini)) Courtesy of SIG Cluster Lifecycle",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77863",
-    "pr_number": 77863,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77874": {
-    "commit": "730bc968b95842022c4c81d607bf6411b459a675",
-    "text": "client-go and kubectl no longer write cached discovery files with world-accessible file permissions",
-    "markdown": "client-go and kubectl no longer write cached discovery files with world-accessible file permissions ([#77874](https://github.com/kubernetes/kubernetes/pull/77874), [@yuchengwu](https://github.com/yuchengwu))",
-    "author": "yuchengwu",
-    "author_url": "https://github.com/yuchengwu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77874",
-    "pr_number": 77874,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "77887": {
-    "commit": "0f78b57fef69c8ec2f3b2f1934d488fea613f6e6",
-    "text": "NodeLocal DNSCache graduating to beta.",
-    "markdown": "NodeLocal DNSCache graduating to beta. ([#77887](https://github.com/kubernetes/kubernetes/pull/77887), [@prameshj](https://github.com/prameshj))",
-    "author": "prameshj",
-    "author_url": "https://github.com/prameshj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77887",
-    "pr_number": 77887,
-    "kinds": ["documentation"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77888": {
-    "commit": "adf6fa69877a50f6cad4ce24214cbf2d41f705a4",
-    "text": "Fix panic logspam when running kubelet in standalone mode.",
-    "markdown": "Fix panic logspam when running kubelet in standalone mode. ([#77888](https://github.com/kubernetes/kubernetes/pull/77888), [@tallclair](https://github.com/tallclair))",
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77888",
-    "pr_number": 77888,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.15.0"
-  },
-  "77889": {
-    "commit": "72f69546142a84590550e37d70260639f8fa3e88",
-    "text": "Support starting Kubernetes on GCE using containerd in COS and Ubuntu with `KUBE_CONTAINER_RUNTIME=containerd`.",
-    "markdown": "Support starting Kubernetes on GCE using containerd in COS and Ubuntu with `KUBE_CONTAINER_RUNTIME=containerd`. ([#77889](https://github.com/kubernetes/kubernetes/pull/77889), [@Random-Liu](https://github.com/Random-Liu)) Courtesy of SIG Cluster Lifecycle, and SIG Node",
-    "author": "Random-Liu",
-    "author_url": "https://github.com/Random-Liu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77889",
-    "pr_number": 77889,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "node"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "77906": {
-    "commit": "5a233ccb84002e18cc8fabc107470a14ac5ebbb8",
-    "text": "Kubelet could be run with no Azure identity now. A sample cloud provider configure is:  `{\"vmType\": \"vmss\", \"useInstanceMetadata\": true}`",
-    "markdown": "Kubelet could be run with no Azure identity now. A sample cloud provider configure is:  `{\"vmType\": \"vmss\", \"useInstanceMetadata\": true}` ([#77906](https://github.com/kubernetes/kubernetes/pull/77906), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77906",
-    "pr_number": 77906,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["azure", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77912": {
-    "commit": "6a9ecf71fdbf9977254d7298c8f073c68683dced",
-    "text": "Fix incorrect azuredisk lun error",
-    "markdown": "Fix incorrect azuredisk lun error ([#77912](https://github.com/kubernetes/kubernetes/pull/77912), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77912",
-    "pr_number": 77912,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["azure", "cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77945": {
-    "commit": "e0f28e5d0f03392e2970742426438c05a1778cb4",
-    "text": "k8s.io/client-go/dynamic/dynamicinformer.NewFilteredDynamicSharedInformerFactory now honours namespace argument",
-    "markdown": "k8s.io/client-go/dynamic/dynamicinformer.NewFilteredDynamicSharedInformerFactory now honours namespace argument ([#77945](https://github.com/kubernetes/kubernetes/pull/77945), [@michaelfig](https://github.com/michaelfig))",
-    "author": "michaelfig",
-    "author_url": "https://github.com/michaelfig",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77945",
-    "pr_number": 77945,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "77946": {
-    "commit": "628e0d8333ea4ef04655ddfd4350561eeaec7262",
-    "text": "Deprecates the kubeadm config upload command as it's replacement is now graduated. Please see `kubeadm init phase upload-config`",
-    "markdown": "Deprecates the kubeadm config upload command as it's replacement is now graduated. Please see `kubeadm init phase upload-config` ([#77946](https://github.com/kubernetes/kubernetes/pull/77946), [@Klaven](https://github.com/Klaven))",
-    "author": "Klaven",
-    "author_url": "https://github.com/Klaven",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77946",
-    "pr_number": 77946,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "77952": {
-    "commit": "27410955e2db0496af0aaa91ac1cdfc8ca20b1fd",
-    "text": "Updates that remove remaining `metadata.finalizers` from  an object that is pending deletion (non-nil metadata.deletionTimestamp) and has no graceful deletion pending (nil or 0 metadata.deletionGracePeriodSeconds) now results in immediate deletion of the object.",
-    "markdown": "Updates that remove remaining `metadata.finalizers` from  an object that is pending deletion (non-nil metadata.deletionTimestamp) and has no graceful deletion pending (nil or 0 metadata.deletionGracePeriodSeconds) now results in immediate deletion of the object. ([#77952](https://github.com/kubernetes/kubernetes/pull/77952), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77952",
-    "pr_number": 77952,
-    "areas": ["apiserver", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "apps", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77975": {
-    "commit": "408735e9dab315bd8f4400fdfa7790ccc3f32ffc",
-    "text": "metrics added to kubelet endpoint 'metrics/probes':\n\n   process_start_time_seconds ",
-    "markdown": "metrics added to kubelet endpoint 'metrics/probes':\n\n   process_start_time_seconds  ([#77975](https://github.com/kubernetes/kubernetes/pull/77975), [@logicalhan](https://github.com/logicalhan))",
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77975",
-    "pr_number": 77975,
-    "areas": ["dependency", "kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "instrumentation", "node"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "77990": {
-    "commit": "82ac5c3cbf04ca63777e340c3bdf9e527dfe0ad0",
-    "text": "Fixed a scheduler racing issue to ensure low priority pods to be unschedulable on the node(s) where high priority pods have `NominatedNodeName` set to the node(s). ",
-    "markdown": "Fixed a scheduler racing issue to ensure low priority pods to be unschedulable on the node(s) where high priority pods have `NominatedNodeName` set to the node(s).  ([#77990](https://github.com/kubernetes/kubernetes/pull/77990), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77990",
-    "pr_number": 77990,
-    "kinds": ["bug", "flake"],
-    "sigs": ["scheduling"],
-    "release_version": "1.15.0"
-  },
-  "77994": {
-    "commit": "05df640f33232e9898ada362371faeabd7cdf395",
-    "text": "Handle resize operation for volume plugins migrated to CSI",
-    "markdown": "Handle resize operation for volume plugins migrated to CSI ([#77994](https://github.com/kubernetes/kubernetes/pull/77994), [@gnufied](https://github.com/gnufied)) Courtesy of SIG API Machinery, SIG Apps, and SIG Storage",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77994",
-    "pr_number": 77994,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "apps", "storage"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "78012": {
-    "commit": "961ea6bd7656302339632c1d4fa5264bcfd52b27",
-    "text": " Upgrade Azure network API version to 2018-07-01, so that EnableTcpReset could be enabled on Azure standard loadbalancer (SLB).",
-    "markdown": " Upgrade Azure network API version to 2018-07-01, so that EnableTcpReset could be enabled on Azure standard loadbalancer (SLB). ([#78012](https://github.com/kubernetes/kubernetes/pull/78012), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78012",
-    "pr_number": 78012,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["azure", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78016": {
-    "commit": "1fba88884be78abe1b231c1c72989b9588751a7a",
-    "text": "kubelet: fix fail to close kubelet-\u003eAPI connections on heartbeat failure when bootstrapping or client certificate rotation is disabled",
-    "markdown": "kubelet: fix fail to close kubelet-\u003eAPI connections on heartbeat failure when bootstrapping or client certificate rotation is disabled ([#78016](https://github.com/kubernetes/kubernetes/pull/78016), [@gaorong](https://github.com/gaorong))",
-    "author": "gaorong",
-    "author_url": "https://github.com/gaorong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78016",
-    "pr_number": 78016,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.15.0"
-  },
-  "78029": {
-    "commit": "0f8009b96d7dc3a7a9af25c0ce8d488f7a190b44",
-    "text": "Active watches of custom resources now terminate properly if the CRD is modified.",
-    "markdown": "Active watches of custom resources now terminate properly if the CRD is modified. ([#78029](https://github.com/kubernetes/kubernetes/pull/78029), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78029",
-    "pr_number": 78029,
-    "areas": ["apiserver", "custom-resources"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "78030": {
-    "commit": "648cb98a44fe8cd47d93bba153dce5b434e0df75",
-    "text": "CoreDNS is now version 1.5.0\n    - A `ready` plugin has been included to report pod readiness\n    - The `proxy` plugin has been deprecated. The `forward` plugin is to be used instead.\n    - CoreDNS fixes the logging now that kubernetes’ client lib switched to klog from glog.",
-    "markdown": "CoreDNS is now version 1.5.0\n    - A `ready` plugin has been included to report pod readiness\n    - The `proxy` plugin has been deprecated. The `forward` plugin is to be used instead.\n    - CoreDNS fixes the logging now that kubernetes’ client lib switched to klog from glog. ([#78030](https://github.com/kubernetes/kubernetes/pull/78030), [@rajansandeep](https://github.com/rajansandeep))",
-    "author": "rajansandeep",
-    "author_url": "https://github.com/rajansandeep",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78030",
-    "pr_number": 78030,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "78053": {
-    "commit": "2f7eaa1ee38b93491e15b98328d02f43b93c3eba",
-    "text": "kubeadm: implement CRI detection for Windows worker nodes",
-    "markdown": "kubeadm: implement CRI detection for Windows worker nodes ([#78053](https://github.com/kubernetes/kubernetes/pull/78053), [@ksubrmnn](https://github.com/ksubrmnn))",
-    "author": "ksubrmnn",
-    "author_url": "https://github.com/ksubrmnn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78053",
-    "pr_number": 78053,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "78059": {
-    "commit": "df4b3c866add70c4b7c1697c6139829eb3a6ccb8",
-    "text": "fix KUBE_SERVER_PLATFORMS null error when cross compiling kubectl for non-linux platform",
-    "markdown": "fix KUBE_SERVER_PLATFORMS null error when cross compiling kubectl for non-linux platform ([#78059](https://github.com/kubernetes/kubernetes/pull/78059), [@figo](https://github.com/figo))",
-    "author": "figo",
-    "author_url": "https://github.com/figo",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78059",
-    "pr_number": 78059,
-    "kinds": ["bug"],
-    "sigs": ["release"],
-    "release_version": "1.15.0"
-  },
-  "78061": {
-    "commit": "b3c1bbb4710f4b178250ae1f5a61be6508501f93",
-    "text": "Introducing new semantic for metric \"volume_operation_total_seconds\" to be the end to end latency of volume provisioning/deletion. Existing metric \"storage_operation_duration_seconds\" will remain untouched however exposed to the following potential issues:\n1. for volume's provisioned/deleted via external provisioner/deleter, \"storage_operation_duration_seconds\" will NOT wait for the external operation to be done before reporting latency metric (effectively close to 0). This will be fixed by using \"volume_operation_total_seconds\" instead\n2. if there's a transient error happened during \"provisioning/deletion\", i.e., a volume is still in-use while a deleteVolume has been called, original \"storage_operation_duration_seconds\" will NOT wait until a volume has been finally deleted before reporting a not accurate latency metric. The newly implemented metric \"volume_operation_total_seconds\", however, wait util a provisioning/deletion operation has been fully executed.\n\nPotential impacts:\nIf an SLO/alert has been defined based on \"volume_operation_total_seconds\", it might get violated because of the more accurate metric might be significantly larger than previously reported. The metric is defined to be a histogram and the new semantic could change the distribution.",
-    "markdown": "Introducing new semantic for metric \"volume_operation_total_seconds\" to be the end to end latency of volume provisioning/deletion. Existing metric \"storage_operation_duration_seconds\" will remain untouched however exposed to the following potential issues:\n1. for volume's provisioned/deleted via external provisioner/deleter, \"storage_operation_duration_seconds\" will NOT wait for the external operation to be done before reporting latency metric (effectively close to 0). This will be fixed by using \"volume_operation_total_seconds\" instead\n2. if there's a transient error happened during \"provisioning/deletion\", i.e., a volume is still in-use while a deleteVolume has been called, original \"storage_operation_duration_seconds\" will NOT wait until a volume has been finally deleted before reporting a not accurate latency metric. The newly implemented metric \"volume_operation_total_seconds\", however, wait util a provisioning/deletion operation has been fully executed.\n\nPotential impacts:\nIf an SLO/alert has been defined based on \"volume_operation_total_seconds\", it might get violated because of the more accurate metric might be significantly larger than previously reported. The metric is defined to be a histogram and the new semantic could change the distribution. ([#78061](https://github.com/kubernetes/kubernetes/pull/78061), [@yuxiangqian](https://github.com/yuxiangqian)) Courtesy of SIG Apps, and SIG Storage",
-    "author": "yuxiangqian",
-    "author_url": "https://github.com/yuxiangqian",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78061",
-    "pr_number": 78061,
-    "kinds": ["feature"],
-    "sigs": ["apps", "storage"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "78135": {
-    "commit": "cd9f430941fa1f27adb57027e0c220272580672b",
-    "text": "Admission webhooks can now register for a single version of a resource (for example, `apps/v1 deployments`) and be called when any other version of that resource is modified (for example `extensions/v1beta1 deployments`). This allows new versions of a resource to be handled by admission webhooks without needing to update every webhook to understand the new version. See the API documentation for the `matchPolicy: Equivalent` option in MutatingWebhookConfiguration and ValidatingWebhookConfiguration types.",
-    "markdown": "Admission webhooks can now register for a single version of a resource (for example, `apps/v1 deployments`) and be called when any other version of that resource is modified (for example `extensions/v1beta1 deployments`). This allows new versions of a resource to be handled by admission webhooks without needing to update every webhook to understand the new version. See the API documentation for the `matchPolicy: Equivalent` option in MutatingWebhookConfiguration and ValidatingWebhookConfiguration types. ([#78135](https://github.com/kubernetes/kubernetes/pull/78135), [@liggitt](https://github.com/liggitt)) Courtesy of SIG API Machinery, and SIG Testing",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78135",
-    "pr_number": 78135,
-    "areas": ["apiserver", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "78137": {
-    "commit": "5cb9405ad8617de7029f5d92fb14d9a1c2107494",
-    "text": "Fix vSphere SAML token auth when using Zones",
-    "markdown": "Fix vSphere SAML token auth when using Zones ([#78137](https://github.com/kubernetes/kubernetes/pull/78137), [@dougm](https://github.com/dougm))",
-    "author": "dougm",
-    "author_url": "https://github.com/dougm",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78137",
-    "pr_number": 78137,
-    "areas": ["cloudprovider", "dependency"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "vmware"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78183": {
-    "commit": "5c314535d1944f1e1fa0fa272de31a1d28baab88",
-    "text": "Ensures that the node-role.kubernetes.io/master taint is applied to the master with NoSchedule on GCE.",
-    "markdown": "Ensures that the node-role.kubernetes.io/master taint is applied to the master with NoSchedule on GCE. ([#78183](https://github.com/kubernetes/kubernetes/pull/78183), [@cheftako](https://github.com/cheftako))",
-    "author": "cheftako",
-    "author_url": "https://github.com/cheftako",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78183",
-    "pr_number": 78183,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "78200": {
-    "commit": "a27d3aef304c5f11a5330bea150edf2d041c1c18",
-    "text": "reverts an aws-ebs volume provisioner optimization as we need to further discuss a viable optimization",
-    "markdown": "reverts an aws-ebs volume provisioner optimization as we need to further discuss a viable optimization ([#78200](https://github.com/kubernetes/kubernetes/pull/78200), [@zhan849](https://github.com/zhan849))",
-    "author": "zhan849",
-    "author_url": "https://github.com/zhan849",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78200",
-    "pr_number": 78200,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["aws", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78242": {
-    "commit": "bb7898ba19084e6f52b86aeb2f00cf6e2b71acba",
-    "text": "Azure cloud provider could now be configured by Kubernetes secrets and a new option `cloudConfigType` is introduced, whose candicate values are `file`, `secret` and `merge` (default is `merge`).\n\n\nSince Azure cloud provider would read Kubernetes secrets, the following RBAC should be configured:\n\n    ---\n    apiVersion: rbac.authorization.k8s.io/v1beta1\n    kind: ClusterRole\n    metadata:\n    labels:\n        kubernetes.io/cluster-service: \"true\"\n    name: system:azure-cloud-provider-secret-getter\n    rules:\n    - apiGroups: [\"\"]\n    resources: [\"secrets\"]\n    verbs:\n    - get\n    ---\n    apiVersion: rbac.authorization.k8s.io/v1beta1\n    kind: ClusterRoleBinding\n    metadata:\n    labels:\n        kubernetes.io/cluster-service: \"true\"\n    name: system:azure-cloud-provider-secret-getter\n    roleRef:\n    apiGroup: rbac.authorization.k8s.io\n    kind: ClusterRole\n    name: system:azure-cloud-provider-secret-getter\n    subjects:\n    - kind: ServiceAccount\n    name: azure-cloud-provider\n    namespace: kube-system",
-    "markdown": "Azure cloud provider could now be configured by Kubernetes secrets and a new option `cloudConfigType` is introduced, whose candicate values are `file`, `secret` and `merge` (default is `merge`).\n\n\nSince Azure cloud provider would read Kubernetes secrets, the following RBAC should be configured:\n\n    ---\n    apiVersion: rbac.authorization.k8s.io/v1beta1\n    kind: ClusterRole\n    metadata:\n    labels:\n        kubernetes.io/cluster-service: \"true\"\n    name: system:azure-cloud-provider-secret-getter\n    rules:\n    - apiGroups: [\"\"]\n    resources: [\"secrets\"]\n    verbs:\n    - get\n    ---\n    apiVersion: rbac.authorization.k8s.io/v1beta1\n    kind: ClusterRoleBinding\n    metadata:\n    labels:\n        kubernetes.io/cluster-service: \"true\"\n    name: system:azure-cloud-provider-secret-getter\n    roleRef:\n    apiGroup: rbac.authorization.k8s.io\n    kind: ClusterRole\n    name: system:azure-cloud-provider-secret-getter\n    subjects:\n    - kind: ServiceAccount\n    name: azure-cloud-provider\n    namespace: kube-system ([#78242](https://github.com/kubernetes/kubernetes/pull/78242), [@feiskyer](https://github.com/feiskyer)) Courtesy of SIG Azure, and SIG Cloud Provider",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78242",
-    "pr_number": 78242,
-    "areas": ["cloudprovider", "dependency"],
-    "kinds": ["feature"],
-    "sigs": ["azure", "cloud-provider"],
-    "feature": true,
-    "action_required": true,
-    "release_version": "1.15.0"
-  },
-  "78261": {
-    "commit": "78254d555a2958483780818ac83f899ff9aa6296",
-    "text": "Fix broken detection of non-root image user ID",
-    "markdown": "Fix broken detection of non-root image user ID ([#78261](https://github.com/kubernetes/kubernetes/pull/78261), [@tallclair](https://github.com/tallclair))",
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78261",
-    "pr_number": 78261,
-    "areas": ["kubelet", "security"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.15.0"
-  },
-  "78262": {
-    "commit": "bc32307314ab3fcb5ed30979872e2885c33c9e36",
-    "text": "Finalizer Protection for Service LoadBalancers is now added as Alpha (disabled by default). This feature ensures the Service resource is not fully deleted until the correlating load balancer resources are deleted.",
-    "markdown": "Finalizer Protection for Service LoadBalancers is now added as Alpha (disabled by default). This feature ensures the Service resource is not fully deleted until the correlating load balancer resources are deleted. ([#78262](https://github.com/kubernetes/kubernetes/pull/78262), [@MrHohn](https://github.com/MrHohn)) Courtesy of SIG Apps, SIG Cloud Provider, SIG Network, and SIG Testing",
-    "author": "MrHohn",
-    "author_url": "https://github.com/MrHohn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78262",
-    "pr_number": 78262,
-    "areas": ["cloudprovider", "test"],
-    "kinds": ["feature"],
-    "sigs": ["apps", "cloud-provider", "network", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "78272": {
-    "commit": "48848731605845c129945f1de146507ca2e5d8cb",
-    "text": "GCE: Disable the Windows defender to work around a bug that could cause nodes to crash and reboot",
-    "markdown": "GCE: Disable the Windows defender to work around a bug that could cause nodes to crash and reboot ([#78272](https://github.com/kubernetes/kubernetes/pull/78272), [@yujuhong](https://github.com/yujuhong))",
-    "author": "yujuhong",
-    "author_url": "https://github.com/yujuhong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78272",
-    "pr_number": 78272,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "gcp"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78276": {
-    "commit": "f14b128ea6429a34b83578d4c2a69a99296a8d31",
-    "text": "Use zone from node for topology aware aws-ebs volume creation to reduce unnecessary cloud provider calls",
-    "markdown": "Use zone from node for topology aware aws-ebs volume creation to reduce unnecessary cloud provider calls ([#78276](https://github.com/kubernetes/kubernetes/pull/78276), [@zhan849](https://github.com/zhan849))",
-    "author": "zhan849",
-    "author_url": "https://github.com/zhan849",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78276",
-    "pr_number": 78276,
-    "kinds": ["bug"],
-    "sigs": ["aws", "scalability", "storage"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78298": {
-    "commit": "fcc9f166fafde843ad9f9d02325bfcd782aae317",
-    "text": "fix azure retry issue when return 2XX with error",
-    "markdown": "fix azure retry issue when return 2XX with error ([#78298](https://github.com/kubernetes/kubernetes/pull/78298), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78298",
-    "pr_number": 78298,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["azure", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78323": {
-    "commit": "eaece39c8692456e14dad5d908f6fb3c347f0ab3",
-    "text": "The dockershim container runtime now accepts the `docker` runtime handler from a RuntimeClass.",
-    "markdown": "The dockershim container runtime now accepts the `docker` runtime handler from a RuntimeClass. ([#78323](https://github.com/kubernetes/kubernetes/pull/78323), [@tallclair](https://github.com/tallclair)) Courtesy of SIG Node, and SIG Testing",
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78323",
-    "pr_number": 78323,
-    "areas": ["kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["node", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "78325": {
-    "commit": "d817a7e9a0e449c997c743526db43c9d718c6364",
-    "text": "The storageVersionHash feature is beta now. \"StorageVersionHash\" is a field in the discovery document of each resource. It allows clients to detect if the storage version of that resource has changed. Its value must be treated as opaque by clients. Only equality comparison on the value is valid.",
-    "markdown": "The storageVersionHash feature is beta now. \"StorageVersionHash\" is a field in the discovery document of each resource. It allows clients to detect if the storage version of that resource has changed. Its value must be treated as opaque by clients. Only equality comparison on the value is valid. ([#78325](https://github.com/kubernetes/kubernetes/pull/78325), [@caesarxuchao](https://github.com/caesarxuchao)) Courtesy of SIG API Machinery, and SIG Testing",
-    "author": "caesarxuchao",
-    "author_url": "https://github.com/caesarxuchao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78325",
-    "pr_number": 78325,
-    "areas": ["apiserver", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "78330": {
-    "commit": "8d5052eddf07bc0f55c00d00af0f034589d9d988",
-    "text": "Add support for Azure Disk plugin to csi-translation-lib",
-    "markdown": "Add support for Azure Disk plugin to csi-translation-lib ([#78330](https://github.com/kubernetes/kubernetes/pull/78330), [@andyzhangx](https://github.com/andyzhangx)) Courtesy of SIG Azure, and SIG Storage",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78330",
-    "pr_number": 78330,
-    "kinds": ["bug", "feature"],
-    "sigs": ["azure", "storage"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "78344": {
-    "commit": "9139adea23e5b268da87f9fc2c32de64c32df682",
-    "text": "Remove deprecated flag --cleanup-iptables from kube-proxy",
-    "markdown": "Remove deprecated flag --cleanup-iptables from kube-proxy ([#78344](https://github.com/kubernetes/kubernetes/pull/78344), [@aramase](https://github.com/aramase))",
-    "author": "aramase",
-    "author_url": "https://github.com/aramase",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78344",
-    "pr_number": 78344,
-    "kinds": ["cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.15.0"
-  },
-  "78356": {
-    "commit": "ade661c7cb684aab4fd18db4928e95ca8ca42338",
-    "text": "Add support for Azure File plugin to csi-translation-lib",
-    "markdown": "Add support for Azure File plugin to csi-translation-lib ([#78356](https://github.com/kubernetes/kubernetes/pull/78356), [@andyzhangx](https://github.com/andyzhangx)) Courtesy of SIG Azure, and SIG Storage",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78356",
-    "pr_number": 78356,
-    "kinds": ["feature"],
-    "sigs": ["azure", "storage"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "78385": {
-    "commit": "3ccb63bb1b4d8a9e9a5e32bdd2750e44a6067ad4",
-    "text": "Allow trailing dots in the externalName of Services of type ExternalName.",
-    "markdown": "Allow trailing dots in the externalName of Services of type ExternalName. ([#78385](https://github.com/kubernetes/kubernetes/pull/78385), [@thz](https://github.com/thz))",
-    "author": "thz",
-    "author_url": "https://github.com/thz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78385",
-    "pr_number": 78385,
-    "kinds": ["bug"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78395": {
-    "commit": "2fb7b6074a05c082b9bbf0429b47bbfc096cbb1b",
-    "text": "Increase log level to 2 for IPVS graceful termination",
-    "markdown": "Increase log level to 2 for IPVS graceful termination ([#78395](https://github.com/kubernetes/kubernetes/pull/78395), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78395",
-    "pr_number": 78395,
-    "areas": ["ipvs"],
-    "kinds": ["cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.15.0"
-  },
-  "78399": {
-    "commit": "cc30c0d9a496dfa19919536fe2950df1cf8dd98f",
-    "text": "ACTION REQUIRED  The deprecated flag --conntrack-max has been removed from kube-proxy. Users of this flag should switch to --conntrack-min and --conntrack-max-per-core instead.",
-    "markdown": "ACTION REQUIRED  The deprecated flag --conntrack-max has been removed from kube-proxy. Users of this flag should switch to --conntrack-min and --conntrack-max-per-core instead. ([#78399](https://github.com/kubernetes/kubernetes/pull/78399), [@rikatz](https://github.com/rikatz)) Courtesy of SIG Cluster Lifecycle, and SIG Network",
-    "author": "rikatz",
-    "author_url": "https://github.com/rikatz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78399",
-    "pr_number": 78399,
-    "areas": ["kubeadm"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "action_required": true,
-    "release_version": "1.15.0"
-  },
-  "78408": {
-    "commit": "de817520826acd44b6098d2a95841850418befc6",
-    "text": "kubeadm: a new command `kubeadm upgrade node` is introduced for upgrading nodes (both secondary control-plane nodes and worker nodes) \nThe command `kubeadm upgrade node config` is now deprecated; use `kubeadm upgrade node` instead.\nThe command `kubeadm upgrade node experimental-control-plane` is now deprecated; use `kubeadm upgrade node` instead.",
-    "markdown": "kubeadm: a new command `kubeadm upgrade node` is introduced for upgrading nodes (both secondary control-plane nodes and worker nodes) \nThe command `kubeadm upgrade node config` is now deprecated; use `kubeadm upgrade node` instead.\nThe command `kubeadm upgrade node experimental-control-plane` is now deprecated; use `kubeadm upgrade node` instead. ([#78408](https://github.com/kubernetes/kubernetes/pull/78408), [@fabriziopandini](https://github.com/fabriziopandini)) Courtesy of SIG Cluster Lifecycle",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78408",
-    "pr_number": 78408,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "78426": {
-    "commit": "b18846cf4cdc1a96d47842bc0ae37d88e7342345",
-    "text": "CustomResourcesDefinition conversion via webhooks is promoted to beta. It requires that spec.preserveUnknownFields is set to false.",
-    "markdown": "CustomResourcesDefinition conversion via webhooks is promoted to beta. It requires that spec.preserveUnknownFields is set to false. ([#78426](https://github.com/kubernetes/kubernetes/pull/78426), [@sttts](https://github.com/sttts)) Courtesy of SIG API Machinery, and SIG Testing",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78426",
-    "pr_number": 78426,
-    "areas": ["custom-resources", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "testing"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "78428": {
-    "commit": "9b14c223758084ddd1c020992d508be9a7db4596",
-    "text": "Fixed panic in kube-proxy when parsing iptables-save output",
-    "markdown": "Fixed panic in kube-proxy when parsing iptables-save output ([#78428](https://github.com/kubernetes/kubernetes/pull/78428), [@luksa](https://github.com/luksa))",
-    "author": "luksa",
-    "author_url": "https://github.com/luksa",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78428",
-    "pr_number": 78428,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.15.0"
-  },
-  "78445": {
-    "commit": "38afd9815e6c9dbfe3db77ead724befb05f3c1e4",
-    "text": "`kubectl scale job`, deprecated since 1.10, has been removed",
-    "markdown": "`kubectl scale job`, deprecated since 1.10, has been removed ([#78445](https://github.com/kubernetes/kubernetes/pull/78445), [@soltysh](https://github.com/soltysh))",
-    "author": "soltysh",
-    "author_url": "https://github.com/soltysh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78445",
-    "pr_number": 78445,
-    "areas": ["kubectl", "test"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78448": {
-    "commit": "225859eb3ee896959e98199003f2ec08eac1fec4",
-    "text": "Promote resource quota scope selector to GA",
-    "markdown": "Promote resource quota scope selector to GA ([#78448](https://github.com/kubernetes/kubernetes/pull/78448), [@ravisantoshgudimetla](https://github.com/ravisantoshgudimetla))",
-    "author": "ravisantoshgudimetla",
-    "author_url": "https://github.com/ravisantoshgudimetla",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78448",
-    "pr_number": 78448,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.15.0"
-  },
-  "78452": {
-    "commit": "44a3bf47550e129cfb4fa13b3bca2c819ca936cd",
-    "text": "kubeadm: flag “--experimental-control-plane” is now deprecated. use “--control-plane” instead\nkubeadm: flag “--experimental-upload-certs” is now deprecated. use “--upload-certs” instead",
-    "markdown": "kubeadm: flag “--experimental-control-plane” is now deprecated. use “--control-plane” instead\nkubeadm: flag “--experimental-upload-certs” is now deprecated. use “--upload-certs” instead ([#78452](https://github.com/kubernetes/kubernetes/pull/78452), [@fabriziopandini](https://github.com/fabriziopandini)) Courtesy of SIG Cluster Lifecycle",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78452",
-    "pr_number": 78452,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "78453": {
-    "commit": "15a01d5fa58266ee8957a48f8ea72ea435890b05",
-    "text": "CustomResourceDefinition with invalid regular expression in the pattern field of OpenAPI v3 validation schemas are not considere structural.",
-    "markdown": "CustomResourceDefinition with invalid regular expression in the pattern field of OpenAPI v3 validation schemas are not considere structural. ([#78453](https://github.com/kubernetes/kubernetes/pull/78453), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78453",
-    "pr_number": 78453,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "78475": {
-    "commit": "8362e9c4f9646aaf045a4d213835610f25348f76",
-    "text": "Till this release, iscsi plugin was waiting 10 seconds for a path to appear in the device list. However this timeout is not enough or less than default device discovery timeout in most of the systems which cause certain device to be not accounted for the volume. This timeout has been lifted to 30seconds from this release and it should help to avoid mount issues due to device discovery.",
-    "markdown": "Till this release, iscsi plugin was waiting 10 seconds for a path to appear in the device list. However this timeout is not enough or less than default device discovery timeout in most of the systems which cause certain device to be not accounted for the volume. This timeout has been lifted to 30seconds from this release and it should help to avoid mount issues due to device discovery. ([#78475](https://github.com/kubernetes/kubernetes/pull/78475), [@humblec](https://github.com/humblec))",
-    "author": "humblec",
-    "author_url": "https://github.com/humblec",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78475",
-    "pr_number": 78475,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.15.0"
-  },
-  "78497": {
-    "commit": "a7a28979b1d0b7740c0bc02fef7e25ef937fdc41",
-    "text": "Supports configure accessLogs for AWS NLB",
-    "markdown": "Supports configure accessLogs for AWS NLB ([#78497](https://github.com/kubernetes/kubernetes/pull/78497), [@M00nF1sh](https://github.com/M00nF1sh)) Courtesy of SIG Cloud Provider",
-    "author": "M00nF1sh",
-    "author_url": "https://github.com/M00nF1sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78497",
-    "pr_number": 78497,
-    "areas": ["cloudprovider"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.15.0"
-  },
-  "78503": {
-    "commit": "9f85c5c4b56940f88cb854615efad4e9f0c66f20",
-    "text": "Horizontal Pod Autoscaling can now scale targets up even when one or more metrics are invalid/unavailable as long as one metric indicates a scale up should occur.",
-    "markdown": "Horizontal Pod Autoscaling can now scale targets up even when one or more metrics are invalid/unavailable as long as one metric indicates a scale up should occur. ([#78503](https://github.com/kubernetes/kubernetes/pull/78503), [@gjtempleton](https://github.com/gjtempleton))",
-    "author": "gjtempleton",
-    "author_url": "https://github.com/gjtempleton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78503",
-    "pr_number": 78503,
-    "kinds": ["cleanup"],
-    "sigs": ["apps", "autoscaling"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78504": {
-    "commit": "6b6bdc760a6a23060ce20f9458e93646fe8c038e",
-    "text": "Deprecate kubelet cAdvisor json endpoints",
-    "markdown": "Deprecate kubelet cAdvisor json endpoints ([#78504](https://github.com/kubernetes/kubernetes/pull/78504), [@dashpole](https://github.com/dashpole))",
-    "author": "dashpole",
-    "author_url": "https://github.com/dashpole",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78504",
-    "pr_number": 78504,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.15.0"
-  },
-  "78505": {
-    "commit": "bada1c6b1eef959825c3dca1d3944e1ac4c31184",
-    "text": "Added objectSelector to admission webhook configurations. objectSelector is evaluated the oldObject and newObject that would be sent to the webhook, and is considered to match if either object matches the selector. A null object (oldObject in the case of create, or newObject in the case of delete) or an object that cannot have labels (like a DeploymentRollback or a PodProxyOptions object) is not considered to match. Use the object selector only if the webhook is opt-in, because end users may skip the admission webhook by setting the labels.",
-    "markdown": "Added objectSelector to admission webhook configurations. objectSelector is evaluated the oldObject and newObject that would be sent to the webhook, and is considered to match if either object matches the selector. A null object (oldObject in the case of create, or newObject in the case of delete) or an object that cannot have labels (like a DeploymentRollback or a PodProxyOptions object) is not considered to match. Use the object selector only if the webhook is opt-in, because end users may skip the admission webhook by setting the labels. ([#78505](https://github.com/kubernetes/kubernetes/pull/78505), [@caesarxuchao](https://github.com/caesarxuchao))",
-    "author": "caesarxuchao",
-    "author_url": "https://github.com/caesarxuchao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78505",
-    "pr_number": 78505,
-    "areas": ["admission-control", "apiserver", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78507": {
-    "commit": "21c9c7a6ab4786fadb4056f0a49f1969acc340db",
-    "text": "Windows nodes on GCE now have the Windows firewall enabled by default.",
-    "markdown": "Windows nodes on GCE now have the Windows firewall enabled by default. ([#78507](https://github.com/kubernetes/kubernetes/pull/78507), [@pjh](https://github.com/pjh))",
-    "author": "pjh",
-    "author_url": "https://github.com/pjh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78507",
-    "pr_number": 78507,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "gcp", "windows"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78522": {
-    "commit": "304a2b191d550cea615687b68c5ff0da4ca5bd2c",
-    "text": "StorageOS volumes now show correct mount information (node and mount time) in the StorageOS administration CLI and UI.",
-    "markdown": "StorageOS volumes now show correct mount information (node and mount time) in the StorageOS administration CLI and UI. ([#78522](https://github.com/kubernetes/kubernetes/pull/78522), [@croomes](https://github.com/croomes))",
-    "author": "croomes",
-    "author_url": "https://github.com/croomes",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78522",
-    "pr_number": 78522,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.15.0"
-  },
-  "78528": {
-    "commit": "d910055d35314f877786074b00b678e6387172ff",
-    "text": "Windows Kubelet nodes will now correctly search the default location for Docker credentials (`%USERPROFILE%\\.docker\\config.json`) when pulling images from a private registry (https://kubernetes.io/docs/concepts/containers/images/\u0026#35;configuring-nodes-to-authenticate-to-a-private-registry)",
-    "markdown": "Windows Kubelet nodes will now correctly search the default location for Docker credentials (`%USERPROFILE%\\.docker\\config.json`) when pulling images from a private registry (https://kubernetes.io/docs/concepts/containers/images/\u0026#35;configuring-nodes-to-authenticate-to-a-private-registry) ([#78528](https://github.com/kubernetes/kubernetes/pull/78528), [@bclau](https://github.com/bclau))",
-    "author": "bclau",
-    "author_url": "https://github.com/bclau",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78528",
-    "pr_number": 78528,
-    "kinds": ["bug"],
-    "sigs": ["windows"],
-    "release_version": "1.15.0"
-  },
-  "78545": {
-    "commit": "16aeec248538144517b6463c4414646f4b2eeeaf",
-    "text": "kubeadm: revert the CoreDNS version to 1.3.1",
-    "markdown": "kubeadm: revert the CoreDNS version to 1.3.1 ([#78545](https://github.com/kubernetes/kubernetes/pull/78545), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78545",
-    "pr_number": 78545,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "78581": {
-    "commit": "ac458e9d89df895424586f8c135b86c4ee07dc91",
-    "text": "Route controller now respects rate limiting to the cloud provider on deletion, previously it was only for create.",
-    "markdown": "Route controller now respects rate limiting to the cloud provider on deletion, previously it was only for create. ([#78581](https://github.com/kubernetes/kubernetes/pull/78581), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78581",
-    "pr_number": 78581,
-    "kinds": ["bug"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78594": {
-    "commit": "acb321e614d21dd7cb8ea55265a9f55363140c39",
-    "text": "Fixes a memory leak in Kubelet on Windows caused by not not closing containers when fetching container metrics",
-    "markdown": "Fixes a memory leak in Kubelet on Windows caused by not not closing containers when fetching container metrics ([#78594](https://github.com/kubernetes/kubernetes/pull/78594), [@benmoss](https://github.com/benmoss))",
-    "author": "benmoss",
-    "author_url": "https://github.com/benmoss",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78594",
-    "pr_number": 78594,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "windows"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78595": {
-    "commit": "333081e79c79b779063935fdbec00452c7511f2b",
-    "text": "Fix issue with kubelet waiting on invalid devicepath on AWS",
-    "markdown": "Fix issue with kubelet waiting on invalid devicepath on AWS ([#78595](https://github.com/kubernetes/kubernetes/pull/78595), [@gnufied](https://github.com/gnufied))",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78595",
-    "pr_number": 78595,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.15.0"
-  },
-  "78608": {
-    "commit": "ca12f110470c6c649202f5229daeb044143f9451",
-    "text": "Fix admission metrics histogram bucket sizes to cover 25ms to ~2.5 seconds.",
-    "markdown": "Fix admission metrics histogram bucket sizes to cover 25ms to ~2.5 seconds. ([#78608](https://github.com/kubernetes/kubernetes/pull/78608), [@jpbetz](https://github.com/jpbetz))",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78608",
-    "pr_number": 78608,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "78611": {
-    "commit": "9c807d483e4321101a203a5b25b35d5d61d69a55",
-    "text": "Fix error handling for loading initCfg in kubeadm upgrade and apply",
-    "markdown": "Fix error handling for loading initCfg in kubeadm upgrade and apply ([#78611](https://github.com/kubernetes/kubernetes/pull/78611), [@odinuge](https://github.com/odinuge))",
-    "author": "odinuge",
-    "author_url": "https://github.com/odinuge",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78611",
-    "pr_number": 78611,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  },
-  "78612": {
-    "commit": "f532d5ca4b792948b9471afb8373ed098eb1a5ad",
-    "text": "Windows kube-proxy will wait for HNS network creation on start",
-    "markdown": "Windows kube-proxy will wait for HNS network creation on start ([#78612](https://github.com/kubernetes/kubernetes/pull/78612), [@ksubrmnn](https://github.com/ksubrmnn))",
-    "author": "ksubrmnn",
-    "author_url": "https://github.com/ksubrmnn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78612",
-    "pr_number": 78612,
-    "kinds": ["bug"],
-    "sigs": ["network", "windows"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78668": {
-    "commit": "3cd41a5a0bc4bbdf156bf925316713a1c41d634a",
-    "text": "kube-up.sh scripts now disable the KubeletPodResources feature for Windows nodes, due to issue \u0026#35;78628.",
-    "markdown": "kube-up.sh scripts now disable the KubeletPodResources feature for Windows nodes, due to issue \u0026#35;78628. ([#78668](https://github.com/kubernetes/kubernetes/pull/78668), [@mtaufen](https://github.com/mtaufen))",
-    "author": "mtaufen",
-    "author_url": "https://github.com/mtaufen",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78668",
-    "pr_number": 78668,
-    "kinds": ["failing-test"],
-    "sigs": ["cluster-lifecycle", "node", "windows"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78691": {
-    "commit": "32ec6c212ec9415f604ffc1f4c1f29b782968ff1",
-    "text": "Revert the CoreDNS version to 1.3.1",
-    "markdown": "Revert the CoreDNS version to 1.3.1 ([#78691](https://github.com/kubernetes/kubernetes/pull/78691), [@rajansandeep](https://github.com/rajansandeep))",
-    "author": "rajansandeep",
-    "author_url": "https://github.com/rajansandeep",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78691",
-    "pr_number": 78691,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78696": {
-    "commit": "e168e558f46159e202ca34b6cb7a243110cc41b9",
-    "text": "Revert Promotion of resource quota scope selector to GA",
-    "markdown": "Revert Promotion of resource quota scope selector to GA ([#78696](https://github.com/kubernetes/kubernetes/pull/78696), [@ravisantoshgudimetla](https://github.com/ravisantoshgudimetla))",
-    "author": "ravisantoshgudimetla",
-    "author_url": "https://github.com/ravisantoshgudimetla",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78696",
-    "pr_number": 78696,
-    "kinds": ["cleanup"],
-    "sigs": ["node", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78700": {
-    "commit": "59594523c80160e181ce49bd399b59868e1f6cbc",
-    "text": "fix: retry detach azure disk issue\ntry to only update vm if detach a non-existing disk when got \u003c200, error\u003e after detach disk operation",
-    "markdown": "fix: retry detach azure disk issue\ntry to only update vm if detach a non-existing disk when got \u003c200, error\u003e after detach disk operation ([#78700](https://github.com/kubernetes/kubernetes/pull/78700), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78700",
-    "pr_number": 78700,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["azure", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78713": {
-    "commit": "f3f2b4066b154575823ad8582e47f81e3db847cd",
-    "text": "Fixed a spurious error where update requests to the status subresource of multi-version custom resources would complain about an incorrect API version.",
-    "markdown": "Fixed a spurious error where update requests to the status subresource of multi-version custom resources would complain about an incorrect API version. ([#78713](https://github.com/kubernetes/kubernetes/pull/78713), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78713",
-    "pr_number": 78713,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "bug", "flake"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "78815": {
-    "commit": "5fe910191a6a1a64872e249c4c431b0169d763ed",
-    "text": "CRDs get support for x-kuberntes-int-or-string to allow faithful representation of IntOrString types in CustomResources.",
-    "markdown": "CRDs get support for x-kuberntes-int-or-string to allow faithful representation of IntOrString types in CustomResources. ([#78815](https://github.com/kubernetes/kubernetes/pull/78815), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78815",
-    "pr_number": 78815,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.0"
-  },
-  "78866": {
-    "commit": "d873167e8f816f5f83846bc4587c3047452293bd",
-    "text": "Update Cluster Autoscaler to 1.15.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.15.0",
-    "markdown": "Update Cluster Autoscaler to 1.15.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.15.0 ([#78866](https://github.com/kubernetes/kubernetes/pull/78866), [@losipiuk](https://github.com/losipiuk))",
-    "author": "losipiuk",
-    "author_url": "https://github.com/losipiuk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78866",
-    "pr_number": 78866,
-    "kinds": ["bug"],
-    "sigs": ["autoscaling", "cluster-lifecycle", "gcp"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78876": {
-    "commit": "8de1569ddae62e8fab559fe6bd210a5d6100a277",
-    "text": "vSphere: allow SAML token delegation (required for Zones support)",
-    "markdown": "vSphere: allow SAML token delegation (required for Zones support) ([#78876](https://github.com/kubernetes/kubernetes/pull/78876), [@dougm](https://github.com/dougm))",
-    "author": "dougm",
-    "author_url": "https://github.com/dougm",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78876",
-    "pr_number": 78876,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "vmware"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78885": {
-    "commit": "0a9373b89ad88a6e72be5208e949d50c1216d95a",
-    "text": "Resolves spurious rollouts of workload controllers when upgrading the API server, due to incorrect defaulting of an alpha procMount field in pods",
-    "markdown": "Resolves spurious rollouts of workload controllers when upgrading the API server, due to incorrect defaulting of an alpha procMount field in pods ([#78885](https://github.com/kubernetes/kubernetes/pull/78885), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78885",
-    "pr_number": 78885,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.0"
-  },
-  "78915": {
-    "commit": "4f29960cb2058c4f556bf1e5ba49c9a29a501e70",
-    "text": "kubeadm: implement retry logic for certain ConfigMap failures when joining nodes",
-    "markdown": "kubeadm: implement retry logic for certain ConfigMap failures when joining nodes ([#78915](https://github.com/kubernetes/kubernetes/pull/78915), [@ereslibre](https://github.com/ereslibre))",
-    "author": "ereslibre",
-    "author_url": "https://github.com/ereslibre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78915",
-    "pr_number": 78915,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.0"
-  }
-}
diff --git a/src/assets/release-notes-1.15.1.json b/src/assets/release-notes-1.15.1.json
deleted file mode 100644
index 51e1513a..00000000
--- a/src/assets/release-notes-1.15.1.json
+++ /dev/null
@@ -1,182 +0,0 @@
-{
-  "75234": {
-    "commit": "b812eaa172804739283e6e8723cbca3ed293e7ff",
-    "text": "fix pod stuck issue due to corrupt mnt point in flexvol plugin, call Unmount if PathExists returns any error",
-    "markdown": "fix pod stuck issue due to corrupt mnt point in flexvol plugin, call Unmount if PathExists returns any error ([#75234](https://github.com/kubernetes/kubernetes/pull/75234), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75234",
-    "pr_number": 75234,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "storage"],
-    "duplicate": true,
-    "release_version": "1.15.1"
-  },
-  "78999": {
-    "commit": "e8462b5b5dc2584fdcd18e6bcfe9f1e4d970a529",
-    "text": "Fix a string comparison bug in IPVS graceful termination where UDP real servers are not deleted.",
-    "markdown": "Fix a string comparison bug in IPVS graceful termination where UDP real servers are not deleted. ([#78999](https://github.com/kubernetes/kubernetes/pull/78999), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78999",
-    "pr_number": 78999,
-    "areas": ["ipvs"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.15.1"
-  },
-  "79073": {
-    "commit": "4ea586a00878ece7b17e47272f8c71d78a4bf4e4",
-    "text": "Remove pids cgroup controller requirement when related feature gates are disabled\n",
-    "markdown": "Remove pids cgroup controller requirement when related feature gates are disabled\n   ([#79073](https://github.com/kubernetes/kubernetes/pull/79073), [@rafatio](https://github.com/rafatio))",
-    "author": "rafatio",
-    "author_url": "https://github.com/rafatio",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79073",
-    "pr_number": 79073,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.15.1"
-  },
-  "79094": {
-    "commit": "a29b88ba3722de4e7f4f699736b4d0cf2722d7ba",
-    "text": "fix kubelet fail to delete orphaned pod directory when the kubelet's pods directory (default is \"/var/lib/kubelet/pods\") symbolically links to another disk device's directory",
-    "markdown": "fix kubelet fail to delete orphaned pod directory when the kubelet's pods directory (default is \"/var/lib/kubelet/pods\") symbolically links to another disk device's directory ([#79094](https://github.com/kubernetes/kubernetes/pull/79094), [@gaorong](https://github.com/gaorong))",
-    "author": "gaorong",
-    "author_url": "https://github.com/gaorong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79094",
-    "pr_number": 79094,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.15.1"
-  },
-  "79114": {
-    "commit": "1bb90b5835bbbf87febc26f711f6144394bea513",
-    "text": "The CRD handler now properly re-creates stale CR storage to reflect CRD update.",
-    "markdown": "The CRD handler now properly re-creates stale CR storage to reflect CRD update. ([#79114](https://github.com/kubernetes/kubernetes/pull/79114), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79114",
-    "pr_number": 79114,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.1"
-  },
-  "79326": {
-    "commit": "5b2266b9262a1dccf21118228194a2b272860c7f",
-    "text": "Fix remove the etcd member from the cluster during a kubeadm reset.",
-    "markdown": "Fix remove the etcd member from the cluster during a kubeadm reset. ([#79326](https://github.com/kubernetes/kubernetes/pull/79326), [@bradbeam](https://github.com/bradbeam))",
-    "author": "bradbeam",
-    "author_url": "https://github.com/bradbeam",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79326",
-    "pr_number": 79326,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.1"
-  },
-  "79451": {
-    "commit": "09a05896f59679ef90f9675c185b7268786e55e4",
-    "text": "Fix a bug where kubelet would not retry pod sandbox creation when the restart policy of the pod is Never",
-    "markdown": "Fix a bug where kubelet would not retry pod sandbox creation when the restart policy of the pod is Never ([#79451](https://github.com/kubernetes/kubernetes/pull/79451), [@yujuhong](https://github.com/yujuhong))",
-    "author": "yujuhong",
-    "author_url": "https://github.com/yujuhong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79451",
-    "pr_number": 79451,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.15.1"
-  },
-  "79498": {
-    "commit": "8d506eacb6efb2a5d2b3180d68e8bbaa6a914c56",
-    "text": "kubeadm: fix the bug that \"--cri-socket\" flag does not work for `kubeadm reset`",
-    "markdown": "kubeadm: fix the bug that \"--cri-socket\" flag does not work for `kubeadm reset` ([#79498](https://github.com/kubernetes/kubernetes/pull/79498), [@SataQiu](https://github.com/SataQiu))",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79498",
-    "pr_number": 79498,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.1"
-  },
-  "79514": {
-    "commit": "93c5b9473c38947861a0bba99111430260c9ac30",
-    "text": "Default resourceGroup should be used when the value of annotation azure-load-balancer-resource-group is an empty string.",
-    "markdown": "Default resourceGroup should be used when the value of annotation azure-load-balancer-resource-group is an empty string. ([#79514](https://github.com/kubernetes/kubernetes/pull/79514), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79514",
-    "pr_number": 79514,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["azure", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.1"
-  },
-  "79529": {
-    "commit": "7795e24751d32b388328ac10a35cb4ac693a9b77",
-    "text": "changes timeout value in csi plugin from 15s to 2min which fixes the timeout issue",
-    "markdown": "changes timeout value in csi plugin from 15s to 2min which fixes the timeout issue ([#79529](https://github.com/kubernetes/kubernetes/pull/79529), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79529",
-    "pr_number": 79529,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.15.1"
-  },
-  "79534": {
-    "commit": "c0f1f9f05278dce793844d69e3443ab0669ad6fe",
-    "text": "Fix possible fd leak and closing of dirs in doSafeMakeDir ",
-    "markdown": "Fix possible fd leak and closing of dirs in doSafeMakeDir  ([#79534](https://github.com/kubernetes/kubernetes/pull/79534), [@odinuge](https://github.com/odinuge))",
-    "author": "odinuge",
-    "author_url": "https://github.com/odinuge",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79534",
-    "pr_number": 79534,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.15.1"
-  },
-  "79636": {
-    "commit": "f44f1c0a698c6a0ee45f6c535e9fb1dc185ba7f1",
-    "text": "Fixes a bug in openapi published for custom resources using x-kubernetes-preserve-unknown-fields extensions, so that kubectl will allow sending unknown fields for that portion of the object.",
-    "markdown": "Fixes a bug in openapi published for custom resources using x-kubernetes-preserve-unknown-fields extensions, so that kubectl will allow sending unknown fields for that portion of the object. ([#79636](https://github.com/kubernetes/kubernetes/pull/79636), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79636",
-    "pr_number": 79636,
-    "areas": ["custom-resources", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.1"
-  },
-  "79677": {
-    "commit": "4485c6f18cee9a5d3c3b4e523bd27972b1b53892",
-    "text": "kubeadm: implement support for concurrent add/remove of stacked etcd members",
-    "markdown": "kubeadm: implement support for concurrent add/remove of stacked etcd members ([#79677](https://github.com/kubernetes/kubernetes/pull/79677), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79677",
-    "pr_number": 79677,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.1"
-  },
-  "79895": {
-    "commit": "7e199be93b39d9f87b9cf7a20c0179bb59c5c726",
-    "text": "Resolves an issue serving aggregated APIs backed by services that respond to requests to `/` with non-2xx HTTP responses",
-    "markdown": "Resolves an issue serving aggregated APIs backed by services that respond to requests to `/` with non-2xx HTTP responses ([#79895](https://github.com/kubernetes/kubernetes/pull/79895), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79895",
-    "pr_number": 79895,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.1"
-  }
-}
diff --git a/src/assets/release-notes-1.15.2.json b/src/assets/release-notes-1.15.2.json
deleted file mode 100644
index e1f8712a..00000000
--- a/src/assets/release-notes-1.15.2.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
-  "80436": {
-    "commit": "f6278300bebbb750328ac16ee6dd3aa7d3549568",
-    "text": "Fix CVE-2019-11249: Incomplete fixes for CVE-2019-1002101 and CVE-2019-11246, kubectl cp potential directory traversal",
-    "markdown": "Fix CVE-2019-11249: Incomplete fixes for CVE-2019-1002101 and CVE-2019-11246, kubectl cp potential directory traversal ([#80436](https://github.com/kubernetes/kubernetes/pull/80436), [@M00nF1sh](https://github.com/M00nF1sh))",
-    "author": "M00nF1sh",
-    "author_url": "https://github.com/M00nF1sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80436",
-    "pr_number": 80436,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.15.2"
-  },
-  "80850": {
-    "commit": "f9a243ca99dc1ef47b8b44c1d00458b6c404dd2a",
-    "text": "Fix CVE-2019-11247: API server allows access to custom resources via wrong scope",
-    "markdown": "Fix CVE-2019-11247: API server allows access to custom resources via wrong scope ([#80850](https://github.com/kubernetes/kubernetes/pull/80850), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80850",
-    "pr_number": 80850,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.2"
-  }
-}
diff --git a/src/assets/release-notes-1.15.3.json b/src/assets/release-notes-1.15.3.json
deleted file mode 100644
index 31bd8060..00000000
--- a/src/assets/release-notes-1.15.3.json
+++ /dev/null
@@ -1,261 +0,0 @@
-{
-  "78495": {
-    "commit": "b3a664eeeb98d5caab02a15367bbd8c7eae31621",
-    "text": "Fix kubelet errors in AArch64 with huge page sizes smaller than 1MiB",
-    "markdown": "Fix kubelet errors in AArch64 with huge page sizes smaller than 1MiB ([#78495](https://github.com/kubernetes/kubernetes/pull/78495), [@odinuge](https://github.com/odinuge))",
-    "author": "odinuge",
-    "author_url": "https://github.com/odinuge",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78495",
-    "pr_number": 78495,
-    "areas": ["apiserver", "dependency", "kubectl", "kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli", "node", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.3"
-  },
-  "79920": {
-    "commit": "6eb69c3f19af4f1b2a0b49b7e649c96f11620cb1",
-    "text": "Bugfix: csi plugin supporting raw block that does not need attach mounted failed",
-    "markdown": "Bugfix: csi plugin supporting raw block that does not need attach mounted failed ([#79920](https://github.com/kubernetes/kubernetes/pull/79920), [@cwdsuzhou](https://github.com/cwdsuzhou))",
-    "author": "cwdsuzhou",
-    "author_url": "https://github.com/cwdsuzhou",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79920",
-    "pr_number": 79920,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.15.3"
-  },
-  "80134": {
-    "commit": "3f2d0c735c4a92b03d3dc6460560b88d84f70486",
-    "text": "update to use go 1.12.7",
-    "markdown": "update to use go 1.12.7 ([#80134](https://github.com/kubernetes/kubernetes/pull/80134), [@tao12345666333](https://github.com/tao12345666333))\n\n  Courtesy of SIG Cluster Lifecycle, SIG Release, and SIG Testing",
-    "author": "tao12345666333",
-    "author_url": "https://github.com/tao12345666333",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80134",
-    "pr_number": 80134,
-    "areas": ["release-eng", "test"],
-    "kinds": ["bug", "feature"],
-    "sigs": ["cluster-lifecycle", "release", "testing"],
-    "feature": true,
-    "release_version": "1.15.3"
-  },
-  "80157": {
-    "commit": "95235fc86f671de31cbbca29dbe53de27862d1ca",
-    "text": "Fix a bug that ListOptions.AllowWatchBookmarks wasn't propagating correctly in kube-apiserver.",
-    "markdown": "Fix a bug that ListOptions.AllowWatchBookmarks wasn't propagating correctly in kube-apiserver. ([#80157](https://github.com/kubernetes/kubernetes/pull/80157), [@wojtek-t](https://github.com/wojtek-t))",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80157",
-    "pr_number": 80157,
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery", "scalability"],
-    "duplicate": true,
-    "release_version": "1.15.3"
-  },
-  "80173": {
-    "commit": "a8951d5e3990b02d23bb1536bfe863b12275a414",
-    "text": "make node lease renew interval more heuristic based on node-status-update-frequency in kubelet",
-    "markdown": "make node lease renew interval more heuristic based on node-status-update-frequency in kubelet ([#80173](https://github.com/kubernetes/kubernetes/pull/80173), [@gaorong](https://github.com/gaorong))",
-    "author": "gaorong",
-    "author_url": "https://github.com/gaorong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80173",
-    "pr_number": 80173,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.15.3"
-  },
-  "80176": {
-    "commit": "c2be2fc9124cb75d044f6215be5722bb3db2af8f",
-    "text": "cpuUsageNanoCores is now reported in the Kubelet summary API on Windows nodes",
-    "markdown": "cpuUsageNanoCores is now reported in the Kubelet summary API on Windows nodes ([#80176](https://github.com/kubernetes/kubernetes/pull/80176), [@liyanhui1228](https://github.com/liyanhui1228))",
-    "author": "liyanhui1228",
-    "author_url": "https://github.com/liyanhui1228",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80176",
-    "pr_number": 80176,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "windows"],
-    "duplicate": true,
-    "release_version": "1.15.3"
-  },
-  "80191": {
-    "commit": "dda8703d204c089b2732950f2baa43ab7de77fdc",
-    "text": "Pass-through volume MountOptions to global mount (NodeStageVolume) on the node for CSI",
-    "markdown": "Pass-through volume MountOptions to global mount (NodeStageVolume) on the node for CSI ([#80191](https://github.com/kubernetes/kubernetes/pull/80191), [@davidz627](https://github.com/davidz627))",
-    "author": "davidz627",
-    "author_url": "https://github.com/davidz627",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80191",
-    "pr_number": 80191,
-    "areas": ["test"],
-    "kinds": ["bug", "failing-test"],
-    "sigs": ["storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.3"
-  },
-  "80675": {
-    "commit": "93d6c59069f6824fb05e926ea094966b47ed8b28",
-    "text": "Fix error in `kubeadm join --discovery-file` when using discovery files with embedded credentials",
-    "markdown": "Fix error in `kubeadm join --discovery-file` when using discovery files with embedded credentials ([#80675](https://github.com/kubernetes/kubernetes/pull/80675), [@fabriziopandini](https://github.com/fabriziopandini))",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80675",
-    "pr_number": 80675,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.3"
-  },
-  "80703": {
-    "commit": "857c81641f0449984f61b9aaa7a9117ca1d87069",
-    "text": "Fix public IP not found issues for VMSS nodes",
-    "markdown": "Fix public IP not found issues for VMSS nodes ([#80703](https://github.com/kubernetes/kubernetes/pull/80703), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80703",
-    "pr_number": 80703,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.3"
-  },
-  "80861": {
-    "commit": "80b73a5d4c167c9b76267fb0854a6d129f8d66ea",
-    "text": "kube-addon-manager has been updated to v9.0.2 to fix a bug in leader election (https://github.com/kubernetes/kubernetes/pull/80575)",
-    "markdown": "kube-addon-manager has been updated to v9.0.2 to fix a bug in leader election (https://github.com/kubernetes/kubernetes/pull/80575) ([#80861](https://github.com/kubernetes/kubernetes/pull/80861), [@mborsz](https://github.com/mborsz))",
-    "author": "mborsz",
-    "author_url": "https://github.com/mborsz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80861",
-    "pr_number": 80861,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.3"
-  },
-  "80923": {
-    "commit": "e8f84b28df44830c3b4908c4d4b66204633b73d2",
-    "text": "Reduces GCE PD Node Attach Limits by 1 since the node boot disk is considered an attachable disk",
-    "markdown": "Reduces GCE PD Node Attach Limits by 1 since the node boot disk is considered an attachable disk ([#80923](https://github.com/kubernetes/kubernetes/pull/80923), [@davidz627](https://github.com/davidz627))",
-    "author": "davidz627",
-    "author_url": "https://github.com/davidz627",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80923",
-    "pr_number": 80923,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.15.3"
-  },
-  "80945": {
-    "commit": "31819c55c5f26a9098134187bcc85707cdce75d7",
-    "text": "Fixes validation of VolumeAttachment API objects created with inline volume sources.",
-    "markdown": "Fixes validation of VolumeAttachment API objects created with inline volume sources. ([#80945](https://github.com/kubernetes/kubernetes/pull/80945), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80945",
-    "pr_number": 80945,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.15.3"
-  },
-  "80978": {
-    "commit": "32884077729c30b123806aa5cad4af3bce63631c",
-    "text": "API: the metadata.selfLink field is deprecated in individual and list objects. It will no longer be returned starting in v1.20, and the field will be removed entirely in v1.21.",
-    "markdown": "API: the metadata.selfLink field is deprecated in individual and list objects. It will no longer be returned starting in v1.20, and the field will be removed entirely in v1.21. ([#80978](https://github.com/kubernetes/kubernetes/pull/80978), [@wojtek-t](https://github.com/wojtek-t))",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80978",
-    "pr_number": 80978,
-    "areas": ["apiserver", "code-generation", "kubelet"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "apps", "node"],
-    "duplicate": true,
-    "release_version": "1.15.3"
-  },
-  "81208": {
-    "commit": "6ef1ff39cbf9a310eec2872f888b90a25d3da9d6",
-    "text": "switch to VM Update call in attach/detach disk operation, original CreateOrUpdate call may lead to orphaned VMs or blocked resources",
-    "markdown": "switch to VM Update call in attach/detach disk operation, original CreateOrUpdate call may lead to orphaned VMs or blocked resources ([#81208](https://github.com/kubernetes/kubernetes/pull/81208), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81208",
-    "pr_number": 81208,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.3"
-  },
-  "81233": {
-    "commit": "cba0944b23a9e566b4e3b47340ec8c75eca2f2c7",
-    "text": "Update the GCE windows node image to include hot fixes since July.",
-    "markdown": "Update the GCE windows node image to include hot fixes since July. ([#81233](https://github.com/kubernetes/kubernetes/pull/81233), [@YangLu1031](https://github.com/YangLu1031))",
-    "author": "YangLu1031",
-    "author_url": "https://github.com/YangLu1031",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81233",
-    "pr_number": 81233,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.3"
-  },
-  "81279": {
-    "commit": "540f7f2013d28f4b9cd95b59384666457cc9ac14",
-    "text": "Fix Azure client requests stuck issues on http.StatusTooManyRequests (HTTP Code 429).",
-    "markdown": "Fix Azure client requests stuck issues on http.StatusTooManyRequests (HTTP Code 429). ([#81279](https://github.com/kubernetes/kubernetes/pull/81279), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81279",
-    "pr_number": 81279,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.3"
-  },
-  "81282": {
-    "commit": "9f37ac02d006807dc6d6028495193224b9924ef1",
-    "text": "Fix conflicted cache when the requests are canceled by other Azure operations.",
-    "markdown": "Fix conflicted cache when the requests are canceled by other Azure operations. ([#81282](https://github.com/kubernetes/kubernetes/pull/81282), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81282",
-    "pr_number": 81282,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.3"
-  },
-  "81390": {
-    "commit": "7812423429d660ebd7a99ce612b8b756ed9344d6",
-    "text": "Update to use go 1.12.8",
-    "markdown": "Update to use go 1.12.8 ([#81390](https://github.com/kubernetes/kubernetes/pull/81390), [@cblecker](https://github.com/cblecker))",
-    "author": "cblecker",
-    "author_url": "https://github.com/cblecker",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81390",
-    "pr_number": 81390,
-    "areas": ["cloudprovider", "test"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "release", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.3"
-  },
-  "81522": {
-    "commit": "2d3c76f9091b6bec110a5e63777c332469e0cba2",
-    "text": "Update golang/x/net dependency to bring in fixes for CVE-2019-9512, CVE-2019-9514",
-    "markdown": "Update golang/x/net dependency to bring in fixes for CVE-2019-9512, CVE-2019-9514 ([#81522](https://github.com/kubernetes/kubernetes/pull/81522), [@cblecker](https://github.com/cblecker))",
-    "author": "cblecker",
-    "author_url": "https://github.com/cblecker",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81522",
-    "pr_number": 81522,
-    "areas": ["apiserver", "cloudprovider", "dependency"],
-    "kinds": ["bug"],
-    "sigs": [
-      "api-machinery",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "network",
-      "node",
-      "release"
-    ],
-    "duplicate": true,
-    "release_version": "1.15.3"
-  }
-}
diff --git a/src/assets/release-notes-1.15.4.json b/src/assets/release-notes-1.15.4.json
deleted file mode 100644
index c66ed1b8..00000000
--- a/src/assets/release-notes-1.15.4.json
+++ /dev/null
@@ -1,159 +0,0 @@
-{
-  "79349": {
-    "commit": "3742c7b810eaf97632c99ee6540cc2404c464dc6",
-    "text": "Fix a bug in server printer that could cause kube-apiserver to panic.",
-    "markdown": "Fix a bug in server printer that could cause kube-apiserver to panic. ([#79349](https://github.com/kubernetes/kubernetes/pull/79349), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79349",
-    "pr_number": 79349,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.15.4"
-  },
-  "79851": {
-    "commit": "867206b3e9e577f4810ea3f6842d0dc2547f241c",
-    "text": "Fixed a bug in the CSI metrics that does not return not supported error when a CSI driver does not support metrics.",
-    "markdown": "Fixed a bug in the CSI metrics that does not return not supported error when a CSI driver does not support metrics. ([#79851](https://github.com/kubernetes/kubernetes/pull/79851), [@jparklab](https://github.com/jparklab))",
-    "author": "jparklab",
-    "author_url": "https://github.com/jparklab",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79851",
-    "pr_number": 79851,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.15.4"
-  },
-  "80942": {
-    "commit": "94a63476781a6c65e8ec748b734d477ed2ef5a06",
-    "text": "Fix a bug in the IPVS proxier where virtual servers are not cleaned up even though the corresponding Service object was deleted.",
-    "markdown": "Fix a bug in the IPVS proxier where virtual servers are not cleaned up even though the corresponding Service object was deleted. ([#80942](https://github.com/kubernetes/kubernetes/pull/80942), [@gongguan](https://github.com/gongguan))",
-    "author": "gongguan",
-    "author_url": "https://github.com/gongguan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80942",
-    "pr_number": 80942,
-    "areas": ["ipvs"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.15.4"
-  },
-  "81215": {
-    "commit": "e19af8f15cab6ff74c766619339b947c4a3d6f37",
-    "text": "remove iSCSI volume storage cleartext secrets in logs",
-    "markdown": "remove iSCSI volume storage cleartext secrets in logs ([#81215](https://github.com/kubernetes/kubernetes/pull/81215), [@zouyee](https://github.com/zouyee))",
-    "author": "zouyee",
-    "author_url": "https://github.com/zouyee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81215",
-    "pr_number": 81215,
-    "kinds": ["bug", "flake"],
-    "sigs": ["storage"],
-    "release_version": "1.15.4"
-  },
-  "81266": {
-    "commit": "78aa54348f26e426323815b3a0a8fabf9f47c0bc",
-    "text": "fix: detach azure disk issue using dangling error",
-    "markdown": "fix: detach azure disk issue using dangling error ([#81266](https://github.com/kubernetes/kubernetes/pull/81266), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81266",
-    "pr_number": 81266,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.15.4"
-  },
-  "81411": {
-    "commit": "99d219012701494d2726eccf2e721484932c441a",
-    "text": "Fix VMSS LoadBalancer backend pools so that the network won't be broken when instances are upgraded to latest model",
-    "markdown": "Fix VMSS LoadBalancer backend pools so that the network won't be broken when instances are upgraded to latest model ([#81411](https://github.com/kubernetes/kubernetes/pull/81411), [@nilo19](https://github.com/nilo19))",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81411",
-    "pr_number": 81411,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.4"
-  },
-  "81515": {
-    "commit": "c0aebb0b0adb46cc703c861d6165d449a1923cf8",
-    "text": "openapi now advertises correctly supported patch types for custom resources",
-    "markdown": "openapi now advertises correctly supported patch types for custom resources ([#81515](https://github.com/kubernetes/kubernetes/pull/81515), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81515",
-    "pr_number": 81515,
-    "areas": ["custom-resources"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.4"
-  },
-  "81720": {
-    "commit": "ae2026b7ea037e6bc01a06d223a1775deb2893fe",
-    "text": "fix azure disk naming matching issue due to case sensitive comparison",
-    "markdown": "fix azure disk naming matching issue due to case sensitive comparison ([#81720](https://github.com/kubernetes/kubernetes/pull/81720), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81720",
-    "pr_number": 81720,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.4"
-  },
-  "81747": {
-    "commit": "7b85be021cd2943167cd3d6b7020f44735d9d90b",
-    "text": "Fix `kubectl logs -f` for windows server containers.",
-    "markdown": "Fix `kubectl logs -f` for windows server containers. ([#81747](https://github.com/kubernetes/kubernetes/pull/81747), [@Random-Liu](https://github.com/Random-Liu))",
-    "author": "Random-Liu",
-    "author_url": "https://github.com/Random-Liu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81747",
-    "pr_number": 81747,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "windows"],
-    "duplicate": true,
-    "release_version": "1.15.4"
-  },
-  "82303": {
-    "commit": "d3ddb6f325f47d91273c7cf362cc5d3b17f1f8f9",
-    "text": "Fix a bug in apiserver that could cause a valid update request to be rejected with a precondition check failure.",
-    "markdown": "Fix a bug in apiserver that could cause a valid update request to be rejected with a precondition check failure. ([#82303](https://github.com/kubernetes/kubernetes/pull/82303), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82303",
-    "pr_number": 82303,
-    "areas": ["apiserver", "custom-resources"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.4"
-  },
-  "82384": {
-    "commit": "f71909037e8daafe2231031510f938e9a6f89a9c",
-    "text": "kubectl cp now safely allows unpacking of symlinks that may point outside the destination directory",
-    "markdown": "kubectl cp now safely allows unpacking of symlinks that may point outside the destination directory ([#82384](https://github.com/kubernetes/kubernetes/pull/82384), [@tallclair](https://github.com/tallclair))",
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82384",
-    "pr_number": 82384,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.15.4"
-  },
-  "82666": {
-    "commit": "90e24044fdfe6b7d01c0f293f543d3c0454699aa",
-    "text": "Restores compatibility with kubectl client-side validation when structural schemas are present in a CRD which sets spec.preserveUnknownFields=true",
-    "markdown": "Restores compatibility with kubectl client-side validation when structural schemas are present in a CRD which sets spec.preserveUnknownFields=true ([#82666](https://github.com/kubernetes/kubernetes/pull/82666), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82666",
-    "pr_number": 82666,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.4"
-  }
-}
diff --git a/src/assets/release-notes-1.15.5.json b/src/assets/release-notes-1.15.5.json
deleted file mode 100644
index 067ff505..00000000
--- a/src/assets/release-notes-1.15.5.json
+++ /dev/null
@@ -1,147 +0,0 @@
-{
-  "80465": {
-    "commit": "db4dd7e22b8eeca306d3c6c29f8879e51fe7aedb",
-    "text": "Resolves bottleneck in internal API server communication that can cause increased goroutines and degrade API Server performance",
-    "markdown": "Resolves bottleneck in internal API server communication that can cause increased goroutines and degrade API Server performance ([#80465](https://github.com/kubernetes/kubernetes/pull/80465), [@answer1991](https://github.com/answer1991))",
-    "author": "answer1991",
-    "author_url": "https://github.com/answer1991",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80465",
-    "pr_number": 80465,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.5"
-  },
-  "82314": {
-    "commit": "05e9e45ef3c87f2dc58c15e266edf9c3464e4cda",
-    "text": "Fix the dns suffix search list for GCE window clusters.",
-    "markdown": "Fix the dns suffix search list for GCE window clusters. ([#82314](https://github.com/kubernetes/kubernetes/pull/82314), [@lzang](https://github.com/lzang))",
-    "author": "lzang",
-    "author_url": "https://github.com/lzang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82314",
-    "pr_number": 82314,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.15.5"
-  },
-  "82640": {
-    "commit": "cc5b08e3597f532c16579debf8db988eb86220e7",
-    "text": "fix: azure disk detach failure if node not exists",
-    "markdown": "fix: azure disk detach failure if node not exists ([#82640](https://github.com/kubernetes/kubernetes/pull/82640), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82640",
-    "pr_number": 82640,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.5"
-  },
-  "82831": {
-    "commit": "825e07d4723be6ebfc96d92ba1d3bc82f06cd307",
-    "text": "Update Cluster Autoscaler to 1.15.2 (https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.15.2)",
-    "markdown": "Update Cluster Autoscaler to 1.15.2 (https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.15.2) ([#82831](https://github.com/kubernetes/kubernetes/pull/82831), [@krzysztof-jastrzebski](https://github.com/krzysztof-jastrzebski))",
-    "author": "krzysztof-jastrzebski",
-    "author_url": "https://github.com/krzysztof-jastrzebski",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82831",
-    "pr_number": 82831,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.5"
-  },
-  "82890": {
-    "commit": "0b85c9d7a13d124e87bd85841059c4a56de70f03",
-    "text": "Update Azure load balancer to prevent orphaned public IP addresses",
-    "markdown": "Update Azure load balancer to prevent orphaned public IP addresses ([#82890](https://github.com/kubernetes/kubernetes/pull/82890), [@chewong](https://github.com/chewong))",
-    "author": "chewong",
-    "author_url": "https://github.com/chewong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82890",
-    "pr_number": 82890,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "cloud-provider", "network"],
-    "duplicate": true,
-    "release_version": "1.15.5"
-  },
-  "83027": {
-    "commit": "364e31bbe5761fcac50ed48fae59d38318734243",
-    "text": "Fixes kube-proxy bug accessing self nodeip:port on windows",
-    "markdown": "Fixes kube-proxy bug accessing self nodeip:port on windows ([#83027](https://github.com/kubernetes/kubernetes/pull/83027), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83027",
-    "pr_number": 83027,
-    "areas": ["dependency"],
-    "kinds": ["bug"],
-    "sigs": ["network", "windows"],
-    "duplicate": true,
-    "release_version": "1.15.5"
-  },
-  "83102": {
-    "commit": "5237945e3a23ccca9650b7b7de0fd29aef091098",
-    "text": "Fix aggressive VM calls for Azure VMSS",
-    "markdown": "Fix aggressive VM calls for Azure VMSS ([#83102](https://github.com/kubernetes/kubernetes/pull/83102), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83102",
-    "pr_number": 83102,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.5"
-  },
-  "83139": {
-    "commit": "295906c2b93815425e7d0c394f03b507748a3f66",
-    "text": "Update to go 1.12.10",
-    "markdown": "Update to go 1.12.10 ([#83139](https://github.com/kubernetes/kubernetes/pull/83139), [@cblecker](https://github.com/cblecker))",
-    "author": "cblecker",
-    "author_url": "https://github.com/cblecker",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83139",
-    "pr_number": 83139,
-    "areas": ["release-eng", "test"],
-    "kinds": ["bug"],
-    "sigs": ["release", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.5"
-  },
-  "83261": {
-    "commit": "9a83f46a506124685802559c4a3981f4f415d1de",
-    "text": "Fixes a flaw (CVE-2019-11253) in json/yaml decoding where large or malformed documents could consume excessive server resources. Request bodies for normal API requests (create/delete/update/patch operations of regular resources) are now limited to 3MB.",
-    "markdown": "Fixes a flaw (CVE-2019-11253) in json/yaml decoding where large or malformed documents could consume excessive server resources. Request bodies for normal API requests (create/delete/update/patch operations of regular resources) are now limited to 3MB. ([#83261](https://github.com/kubernetes/kubernetes/pull/83261), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83261",
-    "pr_number": 83261,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.5"
-  },
-  "83333": {
-    "commit": "eae0bfe8c80300a9191d95d8b2f22cb2a8f13a0f",
-    "text": "Fixes a goroutine leak in kube-apiserver when a request times out.",
-    "markdown": "Fixes a goroutine leak in kube-apiserver when a request times out. ([#83333](https://github.com/kubernetes/kubernetes/pull/83333), [@lavalamp](https://github.com/lavalamp))",
-    "author": "lavalamp",
-    "author_url": "https://github.com/lavalamp",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83333",
-    "pr_number": 83333,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.5"
-  },
-  "83790": {
-    "commit": "99e50682dd3cdc920b0a8db149b1e8ef4f3a78fe",
-    "text": "Fixed panic when accessing CustomResources of a CRD with x-kubernetes-int-or-string.",
-    "markdown": "Fixed panic when accessing CustomResources of a CRD with x-kubernetes-int-or-string. ([#83790](https://github.com/kubernetes/kubernetes/pull/83790), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83790",
-    "pr_number": 83790,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.5"
-  }
-}
diff --git a/src/assets/release-notes-1.15.6.json b/src/assets/release-notes-1.15.6.json
deleted file mode 100644
index ea8921e8..00000000
--- a/src/assets/release-notes-1.15.6.json
+++ /dev/null
@@ -1,240 +0,0 @@
-{
-  "83427": {
-    "commit": "22f43ab27520572acaaf392817dff7b673730b05",
-    "text": "Adds a metric apiserver_request_error_total to kube-apiserver. This metric tallies the number of request_errors encountered by verb, group, version, resource, subresource, scope, component, and code.",
-    "markdown": "Adds a metric apiserver_request_error_total to kube-apiserver. This metric tallies the number of request_errors encountered by verb, group, version, resource, subresource, scope, component, and code. ([#83427](https://github.com/kubernetes/kubernetes/pull/83427), [@logicalhan](https://github.com/logicalhan))",
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83427",
-    "pr_number": 83427,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.15.6"
-  },
-  "83591": {
-    "commit": "c3aac8e00076fd2b7c1af2dc221f1c452e868de4",
-    "text": "Add data cache flushing during unmount device for GCE-PD driver in Windows Server.",
-    "markdown": "Add data cache flushing during unmount device for GCE-PD driver in Windows Server. ([#83591](https://github.com/kubernetes/kubernetes/pull/83591), [@jingxu97](https://github.com/jingxu97))",
-    "author": "jingxu97",
-    "author_url": "https://github.com/jingxu97",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83591",
-    "pr_number": 83591,
-    "kinds": ["bug"],
-    "sigs": ["storage", "windows"],
-    "duplicate": true,
-    "release_version": "1.15.6"
-  },
-  "83776": {
-    "commit": "cf477475ed3c0ab29e445f87447f6d5c25eb0ed1",
-    "text": "removed dependency on test/e2e/common from test/e2e/storage/testsuites",
-    "markdown": "removed dependency on test/e2e/common from test/e2e/storage/testsuites ([#83776](https://github.com/kubernetes/kubernetes/pull/83776), [@avalluri](https://github.com/avalluri))",
-    "author": "avalluri",
-    "author_url": "https://github.com/avalluri",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83776",
-    "pr_number": 83776,
-    "areas": ["test"],
-    "kinds": ["cleanup", "failing-test"],
-    "sigs": ["testing"],
-    "release_version": "1.15.6"
-  },
-  "83789": {
-    "commit": "c97fe5036ef3df2967d086711e6c0c405941e14b",
-    "text": "Fixed panic when accessing CustomResources of a CRD with x-kubernetes-int-or-string.",
-    "markdown": "Fixed panic when accessing CustomResources of a CRD with x-kubernetes-int-or-string. ([#83789](https://github.com/kubernetes/kubernetes/pull/83789), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83789",
-    "pr_number": 83789,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.6"
-  },
-  "83911": {
-    "commit": "a48146da9e06991ccb3d8856d78e2ae23645ecb2",
-    "text": "Fixed an issue with informers missing an `Added` event if a recently deleted object was immediately recreated at the same time the informer dropped a watch and relisted.",
-    "markdown": "Fixed an issue with informers missing an `Added` event if a recently deleted object was immediately recreated at the same time the informer dropped a watch and relisted. ([#83911](https://github.com/kubernetes/kubernetes/pull/83911), [@matte21](https://github.com/matte21))",
-    "author": "matte21",
-    "author_url": "https://github.com/matte21",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83911",
-    "pr_number": 83911,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.6"
-  },
-  "83951": {
-    "commit": "4209b9f168d0e34aeb80b2a30a5298a6a491849d",
-    "text": "Fix handling tombstones in pod-disruption-budged controller.",
-    "markdown": "Fix handling tombstones in pod-disruption-budged controller. ([#83951](https://github.com/kubernetes/kubernetes/pull/83951), [@zouyee](https://github.com/zouyee))",
-    "author": "zouyee",
-    "author_url": "https://github.com/zouyee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83951",
-    "pr_number": 83951,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.15.6"
-  },
-  "83956": {
-    "commit": "15a9f8b0b3a46017a3198980c00593d8a3b8f683",
-    "text": "Switched intstr.Type to sized integer to follow API guidelines and improve compatibility with proto libraries",
-    "markdown": "Switched intstr.Type to sized integer to follow API guidelines and improve compatibility with proto libraries ([#83956](https://github.com/kubernetes/kubernetes/pull/83956), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83956",
-    "pr_number": 83956,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.6"
-  },
-  "83968": {
-    "commit": "f2b2d8b7e2f8056d3751d6e93fb4e1c1ad6bba34",
-    "text": "Upgrade to etcd client 3.3.17 to fix bug where etcd client does not parse IPv6 addresses correctly when members are joining, and to fix bug where failover on multi-member etcd cluster fails certificate check on DNS mismatch",
-    "markdown": "Upgrade to etcd client 3.3.17 to fix bug where etcd client does not parse IPv6 addresses correctly when members are joining, and to fix bug where failover on multi-member etcd cluster fails certificate check on DNS mismatch ([#83968](https://github.com/kubernetes/kubernetes/pull/83968), [@jpbetz](https://github.com/jpbetz))",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83968",
-    "pr_number": 83968,
-    "areas": ["apiserver", "cloudprovider", "dependency"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.6"
-  },
-  "84038": {
-    "commit": "c1f40a5310b0abfe9a4fbddc24955360821a324b",
-    "text": "Update Cluster Autoscaler version to 1.16.2 (CA release docs: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.16.2)",
-    "markdown": "Update Cluster Autoscaler version to 1.16.2 (CA release docs: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.16.2) ([#84038](https://github.com/kubernetes/kubernetes/pull/84038), [@losipiuk](https://github.com/losipiuk))",
-    "author": "losipiuk",
-    "author_url": "https://github.com/losipiuk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84038",
-    "pr_number": 84038,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.6"
-  },
-  "84064": {
-    "commit": "e76a12b6cb1b9d77991d3fc390a87945dbcd3c15",
-    "text": "Update to use go1.12.12",
-    "markdown": "Update to use go1.12.12 ([#84064](https://github.com/kubernetes/kubernetes/pull/84064), [@cblecker](https://github.com/cblecker))",
-    "author": "cblecker",
-    "author_url": "https://github.com/cblecker",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84064",
-    "pr_number": 84064,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["release", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.6"
-  },
-  "84156": {
-    "commit": "c3f2a6524ed89bd9b3ded6f491f0d823b11523ae",
-    "text": "Fix kubelet metrics gathering on non-English Windows hosts",
-    "markdown": "Fix kubelet metrics gathering on non-English Windows hosts ([#84156](https://github.com/kubernetes/kubernetes/pull/84156), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84156",
-    "pr_number": 84156,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "windows"],
-    "duplicate": true,
-    "release_version": "1.15.6"
-  },
-  "84175": {
-    "commit": "3d9fbfdd606aaa912c6c2281bac7f3845a6cf183",
-    "text": "Fixed binding of block PersistentVolumes / PersistentVolumeClaims when BlockVolume feature is off.",
-    "markdown": "Fixed binding of block PersistentVolumes / PersistentVolumeClaims when BlockVolume feature is off. ([#84175](https://github.com/kubernetes/kubernetes/pull/84175), [@jsafrane](https://github.com/jsafrane))",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/b1b2751/keps/sig-storage/20191008-raw-block-support.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84175",
-    "pr_number": 84175,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.15.6"
-  },
-  "84223": {
-    "commit": "a7b7e162982905b46eadc55e2344bfaaa744a1ba",
-    "text": "Bumps metrics-server version to v0.3.6 for following bugfix:\n- Don't break metric storage when duplicate pod metrics encountered causing hpa to fail",
-    "markdown": "Bumps metrics-server version to v0.3.6 for following bugfix:\n  - Don't break metric storage when duplicate pod metrics encountered causing hpa to fail ([#84223](https://github.com/kubernetes/kubernetes/pull/84223), [@olagacek](https://github.com/olagacek))",
-    "author": "olagacek",
-    "author_url": "https://github.com/olagacek",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84223",
-    "pr_number": 84223,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.6"
-  },
-  "84321": {
-    "commit": "1d36d8e3e9b444f6e0b5569f7e97efc5e5ba567d",
-    "text": "CSI detach timeout increased from 10 seconds to 2 minutes",
-    "markdown": "CSI detach timeout increased from 10 seconds to 2 minutes ([#84321](https://github.com/kubernetes/kubernetes/pull/84321), [@cduchesne](https://github.com/cduchesne))",
-    "author": "cduchesne",
-    "author_url": "https://github.com/cduchesne",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84321",
-    "pr_number": 84321,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.15.6"
-  },
-  "84465": {
-    "commit": "ed185580bc39ad4316471ad2170de2c9ac0b06a2",
-    "text": "Restores compatibility of kube-scheduler with clusters that do not enable the events.k8s.io/v1beta1 API",
-    "markdown": "Restores compatibility of kube-scheduler with clusters that do not enable the events.k8s.io/v1beta1 API ([#84465](https://github.com/kubernetes/kubernetes/pull/84465), [@yastij](https://github.com/yastij))",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84465",
-    "pr_number": 84465,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.15.6"
-  },
-  "84536": {
-    "commit": "04bd01ee3f8015caf84f538b834896e03ab1ac9a",
-    "text": "azure: Add allow unsafe read from cache",
-    "markdown": "azure: Add allow unsafe read from cache ([#84536](https://github.com/kubernetes/kubernetes/pull/84536), [@aramase](https://github.com/aramase))",
-    "author": "aramase",
-    "author_url": "https://github.com/aramase",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84536",
-    "pr_number": 84536,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.6"
-  },
-  "84960": {
-    "commit": "c7dab3b96220499c9cba75285b3db61a30f79dd9",
-    "text": "kube-apiserver: fixed a bug that could cause a goroutine leak if the apiserver encountered an encoding error serving a watch to a websocket watcher",
-    "markdown": "kube-apiserver: fixed a bug that could cause a goroutine leak if the apiserver encountered an encoding error serving a watch to a websocket watcher ([#84960](https://github.com/kubernetes/kubernetes/pull/84960), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84960",
-    "pr_number": 84960,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.6"
-  },
-  "84963": {
-    "commit": "09185a04091510940228e6b63df0671aaa37606f",
-    "text": "kube-apiserver: Fixed a regression accepting patch requests \u003e 1MB",
-    "markdown": "kube-apiserver: Fixed a regression accepting patch requests \u003e 1MB ([#84963](https://github.com/kubernetes/kubernetes/pull/84963), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84963",
-    "pr_number": 84963,
-    "areas": ["apiserver", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.6"
-  }
-}
diff --git a/src/assets/release-notes-1.15.7.json b/src/assets/release-notes-1.15.7.json
deleted file mode 100644
index b5404a87..00000000
--- a/src/assets/release-notes-1.15.7.json
+++ /dev/null
@@ -1,94 +0,0 @@
-{
-  "84156": {
-    "commit": "7015f71e75f670eb9e7ebd4b5749639d42e20079",
-    "text": "Fix kubelet metrics gathering on non-English Windows hosts",
-    "markdown": "Fix kubelet metrics gathering on non-English Windows hosts ([#84156](https://github.com/kubernetes/kubernetes/pull/84156), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84156",
-    "pr_number": 84156,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "windows"],
-    "duplicate": true,
-    "release_version": "1.15.7"
-  },
-  "84622": {
-    "commit": "388f59dd3167d3d7edde55d9945c73606b93e191",
-    "text": "Change GCP ILB firewall names to contain the \"k8s-fw-\" prefix like the rest of the firewall rules. This is needed for consistency and also for other components to identify the firewall rule as k8s/service-controller managed.",
-    "markdown": "Change GCP ILB firewall names to contain the \"k8s-fw-\" prefix like the rest of the firewall rules. This is needed for consistency and also for other components to identify the firewall rule as k8s/service-controller managed. ([#84622](https://github.com/kubernetes/kubernetes/pull/84622), [@prameshj](https://github.com/prameshj))",
-    "author": "prameshj",
-    "author_url": "https://github.com/prameshj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84622",
-    "pr_number": 84622,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.7"
-  },
-  "84917": {
-    "commit": "62fab2d1764c89cc0d3929114888faa54d174e3e",
-    "text": "fix race condition when attach/delete azure disk in same time",
-    "markdown": "fix race condition when attach/delete azure disk in same time ([#84917](https://github.com/kubernetes/kubernetes/pull/84917), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84917",
-    "pr_number": 84917,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.7"
-  },
-  "85115": {
-    "commit": "6be749574d9801748fa6b0f084f5cb5b1a732a34",
-    "text": "azure: update disk lock logic per vm during attach/detach to allow concurrent updates for different nodes.",
-    "markdown": "azure: update disk lock logic per vm during attach/detach to allow concurrent updates for different nodes. ([#85115](https://github.com/kubernetes/kubernetes/pull/85115), [@aramase](https://github.com/aramase))",
-    "author": "aramase",
-    "author_url": "https://github.com/aramase",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85115",
-    "pr_number": 85115,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.7"
-  },
-  "85158": {
-    "commit": "d92e1ff3eed88c6628696e7802b6cdc573e89a9d",
-    "text": "fix vmss dirty cache issue in disk attach/detach on vmss node",
-    "markdown": "fix vmss dirty cache issue in disk attach/detach on vmss node ([#85158](https://github.com/kubernetes/kubernetes/pull/85158), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85158",
-    "pr_number": 85158,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.7"
-  },
-  "85318": {
-    "commit": "81c4b9a6225cdff896e998ff24595bffe7e03c1f",
-    "text": "fix azure load balancer update dns label issue",
-    "markdown": "fix azure load balancer update dns label issue ([#85318](https://github.com/kubernetes/kubernetes/pull/85318), [@nilo19](https://github.com/nilo19))",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85318",
-    "pr_number": 85318,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.7"
-  },
-  "85327": {
-    "commit": "8a37612cc108ba45e309f5a4d011ab0728ff1c5e",
-    "text": "Ensure health probes are created for local traffic policy UDP services on Azure",
-    "markdown": "Ensure health probes are created for local traffic policy UDP services on Azure ([#85327](https://github.com/kubernetes/kubernetes/pull/85327), [@nilo19](https://github.com/nilo19))",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85327",
-    "pr_number": 85327,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.7"
-  }
-}
diff --git a/src/assets/release-notes-1.15.8.json b/src/assets/release-notes-1.15.8.json
deleted file mode 100644
index 449a5f2f..00000000
--- a/src/assets/release-notes-1.15.8.json
+++ /dev/null
@@ -1,758 +0,0 @@
-{
-  "73363": {
-    "commit": "67647ed5a379ea72d7db7b5461e9802aa1debdbf",
-    "text": "Minimize AWS NLB health check timeout when externalTrafficPolicy set to Local",
-    "markdown": "Minimize AWS NLB health check timeout when externalTrafficPolicy set to Local ([#73363](https://github.com/kubernetes/kubernetes/pull/73363), [@kellycampbell](https://github.com/kellycampbell))",
-    "author": "kellycampbell",
-    "author_url": "https://github.com/kellycampbell",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73363",
-    "pr_number": 73363,
-    "areas": ["cloudprovider", "provider/aws"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.8"
-  },
-  "82462": {
-    "commit": "5373fa3f593a12309ff0435858991dd117bdb6ca",
-    "text": "kube-proxy: Added dual-stack IPv4/IPv6 support to the iptables proxier.",
-    "markdown": "kube-proxy: Added dual-stack IPv4/IPv6 support to the iptables proxier. ([#82462](https://github.com/kubernetes/kubernetes/pull/82462), [@vllry](https://github.com/vllry))\n\n  Courtesy of SIG Network",
-    "author": "vllry",
-    "author_url": "https://github.com/vllry",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82462",
-    "pr_number": 82462,
-    "areas": ["ipvs"],
-    "kinds": ["feature"],
-    "sigs": ["network"],
-    "feature": true,
-    "release_version": "1.15.8"
-  },
-  "83358": {
-    "commit": "0c026656fe1c697594a756e543578ff3eab98060",
-    "text": "Bind kube-dns containers to linux nodes to avoid Windows scheduling",
-    "markdown": "Bind kube-dns containers to linux nodes to avoid Windows scheduling ([#83358](https://github.com/kubernetes/kubernetes/pull/83358), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83358",
-    "pr_number": 83358,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.15.8"
-  },
-  "83364": {
-    "commit": "fc98aa54465dbb31e2260cc51fb753c8eae55a9e",
-    "text": "Bind dns-horizontal containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes",
-    "markdown": "Bind dns-horizontal containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes ([#83364](https://github.com/kubernetes/kubernetes/pull/83364), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83364",
-    "pr_number": 83364,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.15.8"
-  },
-  "83437": {
-    "commit": "9fa1e00be9f4816e18d463f8125a7f20c0d4aee1",
-    "text": "Promote StartupProbe to beta for 1.18 release",
-    "markdown": "Promote StartupProbe to beta for 1.18 release ([#83437](https://github.com/kubernetes/kubernetes/pull/83437), [@matthyx](https://github.com/matthyx))\n\n  Courtesy of SIG Node, SIG Scalability, and SIG Testing",
-    "author": "matthyx",
-    "author_url": "https://github.com/matthyx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83437",
-    "pr_number": 83437,
-    "areas": ["test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["node", "scalability", "testing"],
-    "feature": true,
-    "release_version": "1.15.8"
-  },
-  "84163": {
-    "commit": "daaa0b9c6ba2dbf37cc0ae4464ed8b35521c7b62",
-    "text": "Bug fixes:\nMake sure we include latest packages node \u0026#35;351 (@caseydavenport)",
-    "markdown": "Bug fixes:\n  Make sure we include latest packages node \u0026#35;351 (@caseydavenport) ([#84163](https://github.com/kubernetes/kubernetes/pull/84163), [@david-tigera](https://github.com/david-tigera))",
-    "author": "david-tigera",
-    "author_url": "https://github.com/david-tigera",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84163",
-    "pr_number": 84163,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.8"
-  },
-  "84462": {
-    "commit": "a1fc96f41e943645229379a6d459826123c11c94",
-    "text": "The underlying format of the `CPUManager` state file has changed. Upgrades should be seamless, but any third-party tools that rely on reading the previous format need to be updated.",
-    "markdown": "The underlying format of the `CPUManager` state file has changed. Upgrades should be seamless, but any third-party tools that rely on reading the previous format need to be updated. ([#84462](https://github.com/kubernetes/kubernetes/pull/84462), [@klueska](https://github.com/klueska))\n\n  Courtesy of SIG Node, and SIG Testing",
-    "author": "klueska",
-    "author_url": "https://github.com/klueska",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84462",
-    "pr_number": 84462,
-    "areas": ["kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["node", "testing"],
-    "feature": true,
-    "release_version": "1.15.8"
-  },
-  "84735": {
-    "commit": "7fdefe50026895f74c7f4685ff2de90f712f05f7",
-    "text": "The sample-apiserver aggregated conformance test has updated to use the Kubernetes v1.17.0 sample apiserver",
-    "markdown": "The sample-apiserver aggregated conformance test has updated to use the Kubernetes v1.17.0 sample apiserver ([#84735](https://github.com/kubernetes/kubernetes/pull/84735), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84735",
-    "pr_number": 84735,
-    "areas": ["conformance", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "architecture", "cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.8"
-  },
-  "84951": {
-    "commit": "010291d4dc3f20d4c5f26f7e04704b9db0659b54",
-    "text": "Fixed a regression where the kubelet would fail to update the ready status of pods.",
-    "markdown": "Fixed a regression where the kubelet would fail to update the ready status of pods. ([#84951](https://github.com/kubernetes/kubernetes/pull/84951), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84951",
-    "pr_number": 84951,
-    "areas": ["kubelet"],
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.15.8"
-  },
-  "85279": {
-    "commit": "2796ff85388112d22ca0145d7762be7beb429e07",
-    "text": "New flag `--show-hidden-metrics-for-version` in kube-proxy can be used to show all hidden metrics that deprecated in the previous minor release.",
-    "markdown": "New flag `--show-hidden-metrics-for-version` in kube-proxy can be used to show all hidden metrics that deprecated in the previous minor release. ([#85279](https://github.com/kubernetes/kubernetes/pull/85279), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Network",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85279",
-    "pr_number": 85279,
-    "areas": ["kubeadm"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "feature": true,
-    "release_version": "1.15.8"
-  },
-  "85308": {
-    "commit": "a26fcf5278a7f643c3aa1dce9be60a3240bb1a72",
-    "text": "If a serving certificates param specifies a name that is an IP for an SNI certificate, it will have priority for replying to server connections.",
-    "markdown": "If a serving certificates param specifies a name that is an IP for an SNI certificate, it will have priority for replying to server connections. ([#85308](https://github.com/kubernetes/kubernetes/pull/85308), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85308",
-    "pr_number": 85308,
-    "areas": ["apiserver", "dependency"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.8"
-  },
-  "85511": {
-    "commit": "92a14f4f7ef11c580961f0aebe491d0ff148e191",
-    "text": "Fix a bug in port-forward: named port not working with service",
-    "markdown": "Fix a bug in port-forward: named port not working with service ([#85511](https://github.com/kubernetes/kubernetes/pull/85511), [@oke-py](https://github.com/oke-py))",
-    "author": "oke-py",
-    "author_url": "https://github.com/oke-py",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85511",
-    "pr_number": 85511,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.15.8"
-  },
-  "85512": {
-    "commit": "127c47caf49bec4a59a0500f99b1b1d4cdbf340e",
-    "text": "Remove cluster-monitoring addon",
-    "markdown": "Remove cluster-monitoring addon ([#85512](https://github.com/kubernetes/kubernetes/pull/85512), [@serathius](https://github.com/serathius))",
-    "author": "serathius",
-    "author_url": "https://github.com/serathius",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85512",
-    "pr_number": 85512,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["cleanup", "deprecation"],
-    "sigs": ["cluster-lifecycle", "instrumentation", "scalability", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.8"
-  },
-  "85517": {
-    "commit": "e1c711227691bd2b32519e86513bbad88b65c972",
-    "text": "add kube-proxy flags --ipvs-tcp-timeout, --ipvs-tcpfin-timeout, --ipvs-udp-timeout to configure IPVS connection timeouts.",
-    "markdown": "add kube-proxy flags --ipvs-tcp-timeout, --ipvs-tcpfin-timeout, --ipvs-udp-timeout to configure IPVS connection timeouts. ([#85517](https://github.com/kubernetes/kubernetes/pull/85517), [@andrewsykim](https://github.com/andrewsykim))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Network",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85517",
-    "pr_number": 85517,
-    "areas": ["ipvs", "kubeadm"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "feature": true,
-    "release_version": "1.15.8"
-  },
-  "85703": {
-    "commit": "d4ba10ec808aec7c3529b4b216d428e024861225",
-    "text": "Fix EndpointSlice controller race condition and ensure that it handles external changes to EndpointSlices.",
-    "markdown": "Fix EndpointSlice controller race condition and ensure that it handles external changes to EndpointSlices. ([#85703](https://github.com/kubernetes/kubernetes/pull/85703), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85703",
-    "pr_number": 85703,
-    "kinds": ["bug"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.15.8"
-  },
-  "85747": {
-    "commit": "0ab15832400c762c7203721d551d2072e2951c6d",
-    "text": "Fixes wrong validation result of NetworkPolicy PolicyTypes",
-    "markdown": "Fixes wrong validation result of NetworkPolicy PolicyTypes ([#85747](https://github.com/kubernetes/kubernetes/pull/85747), [@tnqn](https://github.com/tnqn))",
-    "author": "tnqn",
-    "author_url": "https://github.com/tnqn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85747",
-    "pr_number": 85747,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.15.8"
-  },
-  "85771": {
-    "commit": "d0e901807d4c11e725b7b3ae95d312a13d5dd7f8",
-    "text": "kubernetes will try to acquire the iptables lock every 100 msec during 5 seconds instead of every second. This specially useful for environments using kube-proxy in iptables mode with a high churn rate of services.",
-    "markdown": "kubernetes will try to acquire the iptables lock every 100 msec during 5 seconds instead of every second. This specially useful for environments using kube-proxy in iptables mode with a high churn rate of services. ([#85771](https://github.com/kubernetes/kubernetes/pull/85771), [@aojea](https://github.com/aojea))",
-    "author": "aojea",
-    "author_url": "https://github.com/aojea",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85771",
-    "pr_number": 85771,
-    "kinds": ["flake"],
-    "sigs": ["network"],
-    "release_version": "1.15.8"
-  },
-  "85789": {
-    "commit": "6db550d1db70e389463930bccfb4855f7369b815",
-    "text": "OwnerReferences []OwnerReferencematches \nso nil len(OwnerReferences) ==0 \n\nchange:\n if lease.OwnerReferences == nil || len(lease.OwnerReferences) == 0 ==\u003eif len(lease.OwnerReferences) == 0",
-    "markdown": "OwnerReferences []OwnerReferencematches \n  so nil len(OwnerReferences) ==0 \n  \n  change:\n   if lease.OwnerReferences == nil || len(lease.OwnerReferences) == 0 ==\u003eif len(lease.OwnerReferences) == 0 ([#85789](https://github.com/kubernetes/kubernetes/pull/85789), [@ZP-AlwaysWin](https://github.com/ZP-AlwaysWin))",
-    "author": "ZP-AlwaysWin",
-    "author_url": "https://github.com/ZP-AlwaysWin",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85789",
-    "pr_number": 85789,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.15.8"
-  },
-  "85836": {
-    "commit": "8727eef79c679bf40ff0cfa3318dadbc5fd50e24",
-    "text": "Add instructions about how to bring up e2e test cluster",
-    "markdown": "Add instructions about how to bring up e2e test cluster ([#85836](https://github.com/kubernetes/kubernetes/pull/85836), [@YangLu1031](https://github.com/YangLu1031))",
-    "author": "YangLu1031",
-    "author_url": "https://github.com/YangLu1031",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85836",
-    "pr_number": 85836,
-    "kinds": ["documentation"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.8"
-  },
-  "85837": {
-    "commit": "7b792c38e0c9b66413d790236c80367df3094484",
-    "text": "allow for configuration of CoreDNS replica count",
-    "markdown": "allow for configuration of CoreDNS replica count ([#85837](https://github.com/kubernetes/kubernetes/pull/85837), [@pickledrick](https://github.com/pickledrick))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "pickledrick",
-    "author_url": "https://github.com/pickledrick",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85837",
-    "pr_number": 85837,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.15.8"
-  },
-  "85879": {
-    "commit": "814fc34cde8c51ffc83870e18ad703deece63af7",
-    "text": "hollow-node use fake CRI",
-    "markdown": "hollow-node use fake CRI ([#85879](https://github.com/kubernetes/kubernetes/pull/85879), [@gongguan](https://github.com/gongguan))\n\n  Courtesy of SIG Node, and SIG Scalability",
-    "author": "gongguan",
-    "author_url": "https://github.com/gongguan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85879",
-    "pr_number": 85879,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node", "scalability"],
-    "feature": true,
-    "release_version": "1.15.8"
-  },
-  "85903": {
-    "commit": "6a919f2b54bc3fbc047250f0aa8c423f5209c2b5",
-    "text": "kube-apiserver no longer serves the following deprecated APIs:\n- All resources under `apps/v1beta1` and `apps/v1beta2` - use `apps/v1` instead\n- `daemonsets`, `deployments`, `replicasets` resources under `extensions/v1beta1` - use `apps/v1` instead\n- `networkpolicies` resources under `extensions/v1beta1` - use `networking.k8s.io/v1` instead\n- `podsecuritypolicies` resources under `extensions/v1beta1` - use `policy/v1beta1` instead",
-    "markdown": "kube-apiserver no longer serves the following deprecated APIs:\n  - All resources under `apps/v1beta1` and `apps/v1beta2` - use `apps/v1` instead\n  - `daemonsets`, `deployments`, `replicasets` resources under `extensions/v1beta1` - use `apps/v1` instead\n  - `networkpolicies` resources under `extensions/v1beta1` - use `networking.k8s.io/v1` instead\n  - `podsecuritypolicies` resources under `extensions/v1beta1` - use `policy/v1beta1` instead ([#85903](https://github.com/kubernetes/kubernetes/pull/85903), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85903",
-    "pr_number": 85903,
-    "areas": ["apiserver", "test"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["api-machinery", "apps", "cluster-lifecycle", "instrumentation", "testing"],
-    "duplicate": true,
-    "release_version": "1.15.8"
-  },
-  "85944": {
-    "commit": "a233e0115bb56e014f812365f583704075accc7b",
-    "text": "elasticsearch supports automatically setting the advertise address",
-    "markdown": "elasticsearch supports automatically setting the advertise address ([#85944](https://github.com/kubernetes/kubernetes/pull/85944), [@SataQiu](https://github.com/SataQiu))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Instrumentation",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85944",
-    "pr_number": 85944,
-    "kinds": ["bug", "feature"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "feature": true,
-    "release_version": "1.15.8"
-  },
-  "85975": {
-    "commit": "6f61072606155b070b2f79850b008685df3a74ff",
-    "text": "Fix nil pointer dereference in azure cloud provider",
-    "markdown": "Fix nil pointer dereference in azure cloud provider ([#85975](https://github.com/kubernetes/kubernetes/pull/85975), [@ldx](https://github.com/ldx))",
-    "author": "ldx",
-    "author_url": "https://github.com/ldx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85975",
-    "pr_number": 85975,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.8"
-  },
-  "85993": {
-    "commit": "7a50fdb2a635f4f3219f6411fa42d89c65a77800",
-    "text": "fix a regression in kubenet that prevent pods to obtain ip addresses",
-    "markdown": "fix a regression in kubenet that prevent pods to obtain ip addresses ([#85993](https://github.com/kubernetes/kubernetes/pull/85993), [@chendotjs](https://github.com/chendotjs))",
-    "author": "chendotjs",
-    "author_url": "https://github.com/chendotjs",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85993",
-    "pr_number": 85993,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["network", "node"],
-    "duplicate": true,
-    "release_version": "1.15.8"
-  },
-  "86016": {
-    "commit": "078344c12524adfe1f6f742e0d5a21016d65e64b",
-    "text": "Fixes kube-proxy when EndpointSlice feature gate is enabled on Windows.",
-    "markdown": "Fixes kube-proxy when EndpointSlice feature gate is enabled on Windows. ([#86016](https://github.com/kubernetes/kubernetes/pull/86016), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86016",
-    "pr_number": 86016,
-    "kinds": ["bug"],
-    "sigs": ["auth", "network"],
-    "duplicate": true,
-    "release_version": "1.15.8"
-  },
-  "86092": {
-    "commit": "442107b6b96a980e4d26c30a176071092748414e",
-    "text": "kube-proxy no longer modifies shared EndpointSlices.",
-    "markdown": "kube-proxy no longer modifies shared EndpointSlices. ([#86092](https://github.com/kubernetes/kubernetes/pull/86092), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86092",
-    "pr_number": 86092,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.15.8"
-  },
-  "86167": {
-    "commit": "03e90b80ceebc4197eae7cc6b95a59623bd9a51f",
-    "text": "change CounterVec to Counter about  PLEGDiscardEvent",
-    "markdown": "change CounterVec to Counter about  PLEGDiscardEvent ([#86167](https://github.com/kubernetes/kubernetes/pull/86167), [@yiyang5055](https://github.com/yiyang5055))",
-    "author": "yiyang5055",
-    "author_url": "https://github.com/yiyang5055",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86167",
-    "pr_number": 86167,
-    "kinds": ["cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.15.8"
-  },
-  "86168": {
-    "commit": "0d58709016f3358f34a52c12087d707a64cc6cb0",
-    "text": "Resolves performance regression in `kubectl get all` and in client-go discovery clients constructed using `NewDiscoveryClientForConfig` or `NewDiscoveryClientForConfigOrDie`.",
-    "markdown": "Resolves performance regression in `kubectl get all` and in client-go discovery clients constructed using `NewDiscoveryClientForConfig` or `NewDiscoveryClientForConfigOrDie`. ([#86168](https://github.com/kubernetes/kubernetes/pull/86168), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86168",
-    "pr_number": 86168,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.8"
-  },
-  "86210": {
-    "commit": "3ccdad175a80a73c8a756e2f35b7ac1d6a98d106",
-    "text": "The following features are unconditionally enabled and the corresponding `--feature-gates` flags have been removed: `PodPriority`, `TaintNodesByCondition`, `ResourceQuotaScopeSelectors` and `ScheduleDaemonSetPods`",
-    "markdown": "The following features are unconditionally enabled and the corresponding `--feature-gates` flags have been removed: `PodPriority`, `TaintNodesByCondition`, `ResourceQuotaScopeSelectors` and `ScheduleDaemonSetPods` ([#86210](https://github.com/kubernetes/kubernetes/pull/86210), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86210",
-    "pr_number": 86210,
-    "kinds": ["cleanup"],
-    "sigs": ["apps", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.15.8"
-  },
-  "86216": {
-    "commit": "e622579b143730b89d471ecd4e5f6761e9c37d86",
-    "text": "kubeadm: fix potential panic when executing \"kubeadm reset\" with a corrupted kubelet.conf file",
-    "markdown": "kubeadm: fix potential panic when executing \"kubeadm reset\" with a corrupted kubelet.conf file ([#86216](https://github.com/kubernetes/kubernetes/pull/86216), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86216",
-    "pr_number": 86216,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.8"
-  },
-  "86251": {
-    "commit": "49bc6966149a92e02aaee0245881c6a4a83dc007",
-    "text": "New metric kubelet_pleg_last_seen_seconds to aid diagnosis of PLEG not healthy issues.",
-    "markdown": "New metric kubelet_pleg_last_seen_seconds to aid diagnosis of PLEG not healthy issues. ([#86251](https://github.com/kubernetes/kubernetes/pull/86251), [@bboreham](https://github.com/bboreham))\n\n  Courtesy of SIG Node",
-    "author": "bboreham",
-    "author_url": "https://github.com/bboreham",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86251",
-    "pr_number": 86251,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.15.8"
-  },
-  "86266": {
-    "commit": "f06a3800ed7819fa68aa30054b4cd52619a4cbde",
-    "text": "azure cloud provider cache TTL is configurable, list of the azure cloud provider is as following:\n- \"availabilitySetNodesCacheTTLInSeconds\"\n- \"vmssCacheTTLInSeconds\"\n- \"vmssVirtualMachinesCacheTTLInSeconds\"\n- \"vmCacheTTLInSeconds\"\n- \"loadBalancerCacheTTLInSeconds\"\n- \"nsgCacheTTLInSeconds\"\n- \"routeTableCacheTTLInSeconds\"",
-    "markdown": "azure cloud provider cache TTL is configurable, list of the azure cloud provider is as following:\n  - \"availabilitySetNodesCacheTTLInSeconds\"\n  - \"vmssCacheTTLInSeconds\"\n  - \"vmssVirtualMachinesCacheTTLInSeconds\"\n  - \"vmCacheTTLInSeconds\"\n  - \"loadBalancerCacheTTLInSeconds\"\n  - \"nsgCacheTTLInSeconds\"\n  - \"routeTableCacheTTLInSeconds\" ([#86266](https://github.com/kubernetes/kubernetes/pull/86266), [@zqingqing1](https://github.com/zqingqing1))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "zqingqing1",
-    "author_url": "https://github.com/zqingqing1",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86266",
-    "pr_number": 86266,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.15.8"
-  },
-  "86277": {
-    "commit": "a57a8f33b323cd002e76c664bf0f91f71df7c927",
-    "text": "Fixed a panic in the kubelet cleaning up pod volumes",
-    "markdown": "Fixed a panic in the kubelet cleaning up pod volumes ([#86277](https://github.com/kubernetes/kubernetes/pull/86277), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86277",
-    "pr_number": 86277,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.15.8"
-  },
-  "86320": {
-    "commit": "eee9dee7b9c0ad926c8619b211902f118feff2fa",
-    "text": "Fixes an issue with kubelet-reported pod status on deleted/recreated pods.",
-    "markdown": "Fixes an issue with kubelet-reported pod status on deleted/recreated pods. ([#86320](https://github.com/kubernetes/kubernetes/pull/86320), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86320",
-    "pr_number": 86320,
-    "areas": ["kubelet"],
-    "kinds": ["bug", "flake"],
-    "sigs": ["node"],
-    "release_version": "1.15.8"
-  },
-  "86329": {
-    "commit": "4a62b3ac6dc7937ea798f2163cb3bb1f1a165f20",
-    "text": "Changed core_pattern on COS nodes to be an absolute path.",
-    "markdown": "Changed core_pattern on COS nodes to be an absolute path. ([#86329](https://github.com/kubernetes/kubernetes/pull/86329), [@mml](https://github.com/mml))",
-    "author": "mml",
-    "author_url": "https://github.com/mml",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86329",
-    "pr_number": 86329,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "node"],
-    "duplicate": true,
-    "release_version": "1.15.8"
-  },
-  "86338": {
-    "commit": "6f4ec3e0d6e1c1e9947091aedcaee882753363d6",
-    "text": "option `preConfiguredBackendPoolLoadBalancerTypes` is added to azure cloud provider for the pre-configured load balancers, possible values: `\"\"`, `\"internal\"`, \"external\"`, `\"all\"`",
-    "markdown": "option `preConfiguredBackendPoolLoadBalancerTypes` is added to azure cloud provider for the pre-configured load balancers, possible values: `\"\"`, `\"internal\"`, \"external\"`, `\"all\"` ([#86338](https://github.com/kubernetes/kubernetes/pull/86338), [@gossion](https://github.com/gossion))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "gossion",
-    "author_url": "https://github.com/gossion",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86338",
-    "pr_number": 86338,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.15.8"
-  },
-  "86351": {
-    "commit": "ac4079a5cc22ced66c868f6fd26f8e2497765e51",
-    "text": "fix: azure data disk should use same key as os disk by default",
-    "markdown": "fix: azure data disk should use same key as os disk by default ([#86351](https://github.com/kubernetes/kubernetes/pull/86351), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86351",
-    "pr_number": 86351,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.8"
-  },
-  "86369": {
-    "commit": "84198bf1206c92a14a5f237c4161d5138886492e",
-    "text": "`AlwaysCheckAllPredicates` is deprecated in scheduler Policy API.",
-    "markdown": "`AlwaysCheckAllPredicates` is deprecated in scheduler Policy API. ([#86369](https://github.com/kubernetes/kubernetes/pull/86369), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86369",
-    "pr_number": 86369,
-    "kinds": ["api-change", "cleanup", "deprecation"],
-    "sigs": ["scheduling"],
-    "release_version": "1.15.8"
-  },
-  "86412": {
-    "commit": "3577447764103cc3204b66307d651ac2be0ec83f",
-    "text": "Fixes issue where AAD token obtained by kubectl is incompatible with on-behalf-of flow and oidc.\nThe audience claim before this fix has \"spn:\" prefix. After this fix, \"spn:\" prefix is omitted.",
-    "markdown": "Fixes issue where AAD token obtained by kubectl is incompatible with on-behalf-of flow and oidc.\n  The audience claim before this fix has \"spn:\" prefix. After this fix, \"spn:\" prefix is omitted. ([#86412](https://github.com/kubernetes/kubernetes/pull/86412), [@weinong](https://github.com/weinong))",
-    "author": "weinong",
-    "author_url": "https://github.com/weinong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86412",
-    "pr_number": 86412,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.8"
-  },
-  "86425": {
-    "commit": "9f841bf42396daf1ed4b7ab47c1e1708bfb028b3",
-    "text": "hollow-node do not use remote CRI anymore",
-    "markdown": "hollow-node do not use remote CRI anymore ([#86425](https://github.com/kubernetes/kubernetes/pull/86425), [@jkaniuk](https://github.com/jkaniuk))",
-    "author": "jkaniuk",
-    "author_url": "https://github.com/jkaniuk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86425",
-    "pr_number": 86425,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "scalability"],
-    "duplicate": true,
-    "release_version": "1.15.8"
-  },
-  "86493": {
-    "commit": "3760d24674d9eac92284c5d19c4a774aceab1b7f",
-    "text": "kubeadm: use bind-address option to configure the kube-controller-manager and kube-scheduler http probes",
-    "markdown": "kubeadm: use bind-address option to configure the kube-controller-manager and kube-scheduler http probes ([#86493](https://github.com/kubernetes/kubernetes/pull/86493), [@aojea](https://github.com/aojea))",
-    "author": "aojea",
-    "author_url": "https://github.com/aojea",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86493",
-    "pr_number": 86493,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.8"
-  },
-  "86496": {
-    "commit": "06fb3eb582187931a22c9fc57024063d91d965f8",
-    "text": "Adds back support for AlwaysCheckAllPredicates flag.",
-    "markdown": "Adds back support for AlwaysCheckAllPredicates flag. ([#86496](https://github.com/kubernetes/kubernetes/pull/86496), [@ahg-g](https://github.com/ahg-g))",
-    "author": "ahg-g",
-    "author_url": "https://github.com/ahg-g",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86496",
-    "pr_number": 86496,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.15.8"
-  },
-  "86502": {
-    "commit": "4f70231ce7736cc748f76526c98955f86c667a41",
-    "text": "Fix issue \u0026#35;85805 about resource not found in azure cloud provider when lb specified in other resource group.",
-    "markdown": "Fix issue \u0026#35;85805 about resource not found in azure cloud provider when lb specified in other resource group. ([#86502](https://github.com/kubernetes/kubernetes/pull/86502), [@levimm](https://github.com/levimm))",
-    "author": "levimm",
-    "author_url": "https://github.com/levimm",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86502",
-    "pr_number": 86502,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.8"
-  },
-  "86515": {
-    "commit": "92e0de32698864f4dce876d79e80386261e3e18c",
-    "text": "Azure global rate limit is switched to per-client. A set of new rate limit configure options are introduced, including routeRateLimit, SubnetsRateLimit, InterfaceRateLimit, RouteTableRateLimit, LoadBalancerRateLimit, PublicIPAddressRateLimit, SecurityGroupRateLimit, VirtualMachineRateLimit, StorageAccountRateLimit, DiskRateLimit, SnapshotRateLimit, VirtualMachineScaleSetRateLimit and VirtualMachineSizeRateLimit.\n\nThe original rate limit options would be default values for those new client's rate limiter.",
-    "markdown": "Azure global rate limit is switched to per-client. A set of new rate limit configure options are introduced, including routeRateLimit, SubnetsRateLimit, InterfaceRateLimit, RouteTableRateLimit, LoadBalancerRateLimit, PublicIPAddressRateLimit, SecurityGroupRateLimit, VirtualMachineRateLimit, StorageAccountRateLimit, DiskRateLimit, SnapshotRateLimit, VirtualMachineScaleSetRateLimit and VirtualMachineSizeRateLimit.\n  \n  The original rate limit options would be default values for those new client's rate limiter. ([#86515](https://github.com/kubernetes/kubernetes/pull/86515), [@feiskyer](https://github.com/feiskyer))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86515",
-    "pr_number": 86515,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.15.8"
-  },
-  "86534": {
-    "commit": "5e31799701123c50025567b8534e1a62dbc0e9f6",
-    "text": "Fixes v1.17.0 regression in --service-cluster-ip-range handling with IPv4 ranges larger than 65536 IP addresses",
-    "markdown": "Fixes v1.17.0 regression in --service-cluster-ip-range handling with IPv4 ranges larger than 65536 IP addresses ([#86534](https://github.com/kubernetes/kubernetes/pull/86534), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86534",
-    "pr_number": 86534,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.15.8"
-  },
-  "86584": {
-    "commit": "180e1c9f5159a22576ee866a82a7dc8c26cc1c41",
-    "text": "Marked scheduler's metrics scheduling_algorithm_predicate_evaluation_seconds and\nscheduling_algorithm_priority_evaluation_seconds as deprecated. Those are replaced by framework_extension_point_duration_seconds[extenstion_point=\"Filter\"] and framework_extension_point_duration_seconds[extenstion_point=\"Score\"] respectively.",
-    "markdown": "Marked scheduler's metrics scheduling_algorithm_predicate_evaluation_seconds and\n  scheduling_algorithm_priority_evaluation_seconds as deprecated. Those are replaced by framework_extension_point_duration_seconds[extenstion_point=\"Filter\"] and framework_extension_point_duration_seconds[extenstion_point=\"Score\"] respectively. ([#86584](https://github.com/kubernetes/kubernetes/pull/86584), [@xiaoanyunfei](https://github.com/xiaoanyunfei))",
-    "author": "xiaoanyunfei",
-    "author_url": "https://github.com/xiaoanyunfei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86584",
-    "pr_number": 86584,
-    "kinds": ["cleanup", "deprecation"],
-    "sigs": ["scheduling"],
-    "release_version": "1.15.8"
-  },
-  "86586": {
-    "commit": "fdfcb00edee127df1f9d27f0b858271ed8959421",
-    "text": "Marked scheduler's scheduling_duration_seconds Summary metric as deprecated",
-    "markdown": "Marked scheduler's scheduling_duration_seconds Summary metric as deprecated ([#86586](https://github.com/kubernetes/kubernetes/pull/86586), [@xiaoanyunfei](https://github.com/xiaoanyunfei))",
-    "author": "xiaoanyunfei",
-    "author_url": "https://github.com/xiaoanyunfei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86586",
-    "pr_number": 86586,
-    "kinds": ["cleanup", "deprecation"],
-    "sigs": ["scheduling"],
-    "release_version": "1.15.8"
-  },
-  "86609": {
-    "commit": "f772c9017c16bbc983e59a19162814629a467cef",
-    "text": "kubeadm: add support for the \"ci/k8s-master\" version label as a replacement for \"ci-cross/*\", which no longer exists.",
-    "markdown": "kubeadm: add support for the \"ci/k8s-master\" version label as a replacement for \"ci-cross/*\", which no longer exists. ([#86609](https://github.com/kubernetes/kubernetes/pull/86609), [@Pensu](https://github.com/Pensu))",
-    "author": "Pensu",
-    "author_url": "https://github.com/Pensu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86609",
-    "pr_number": 86609,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.8"
-  },
-  "86612": {
-    "commit": "40320be679e9bc85632a621dc2fa12af39f37c6d",
-    "text": "fix: azure disk could not mounted on Standard_DC4s/DC2s instances",
-    "markdown": "fix: azure disk could not mounted on Standard_DC4s/DC2s instances ([#86612](https://github.com/kubernetes/kubernetes/pull/86612), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86612",
-    "pr_number": 86612,
-    "areas": ["provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.15.8"
-  },
-  "86646": {
-    "commit": "19ecd690fa1ff0747db6388714c0569dfc2548bd",
-    "text": "For subprotocol negotiation, both client and server protocol is required now.",
-    "markdown": "For subprotocol negotiation, both client and server protocol is required now. ([#86646](https://github.com/kubernetes/kubernetes/pull/86646), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86646",
-    "pr_number": 86646,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "node"],
-    "duplicate": true,
-    "release_version": "1.15.8"
-  },
-  "86693": {
-    "commit": "c72d5fd2556f90669fc3321061dba658cad077cf",
-    "text": "If a required flag is not provided to a command, the user will only see the required flag error message, instead of the entire usage menu.",
-    "markdown": "If a required flag is not provided to a command, the user will only see the required flag error message, instead of the entire usage menu. ([#86693](https://github.com/kubernetes/kubernetes/pull/86693), [@sallyom](https://github.com/sallyom))",
-    "author": "sallyom",
-    "author_url": "https://github.com/sallyom",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86693",
-    "pr_number": 86693,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.15.8"
-  },
-  "86705": {
-    "commit": "09cb73a554d3218f0267ba28de8a5fba47fd6335",
-    "text": "kubeadm: tolerate whitespace when validating certificate authority PEM data in kubeconfig files",
-    "markdown": "kubeadm: tolerate whitespace when validating certificate authority PEM data in kubeconfig files ([#86705](https://github.com/kubernetes/kubernetes/pull/86705), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86705",
-    "pr_number": 86705,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.8"
-  },
-  "86740": {
-    "commit": "b046dbce1332f6d67e8c3adf03a7b62f605b759c",
-    "text": "Azure VMSS/VMSSVM clients now suppress requests on throttling",
-    "markdown": "Azure VMSS/VMSSVM clients now suppress requests on throttling ([#86740](https://github.com/kubernetes/kubernetes/pull/86740), [@feiskyer](https://github.com/feiskyer))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86740",
-    "pr_number": 86740,
-    "areas": ["cloudprovider", "dependency", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.15.8"
-  },
-  "86940": {
-    "commit": "edad4bbfc824215fc254096dfbbd1b2ab8ce6781",
-    "text": "Add VolumeBinder method to FrameworkHandle interface, which allows user to get the volume binder when implementing scheduler framework plugins.",
-    "markdown": "Add VolumeBinder method to FrameworkHandle interface, which allows user to get the volume binder when implementing scheduler framework plugins. ([#86940](https://github.com/kubernetes/kubernetes/pull/86940), [@skilxn-go](https://github.com/skilxn-go))\n\n  Courtesy of SIG Scheduling",
-    "author": "skilxn-go",
-    "author_url": "https://github.com/skilxn-go",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86940",
-    "pr_number": 86940,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.15.8"
-  },
-  "86952": {
-    "commit": "6026a6a1515244855d10a51857af2e8c2fd26f37",
-    "text": "fix kubectl annotate error when local=true is set",
-    "markdown": "fix kubectl annotate error when local=true is set ([#86952](https://github.com/kubernetes/kubernetes/pull/86952), [@zhouya0](https://github.com/zhouya0))",
-    "author": "zhouya0",
-    "author_url": "https://github.com/zhouya0",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86952",
-    "pr_number": 86952,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.15.8"
-  },
-  "87043": {
-    "commit": "240782c7f7ee1a810207b532f970ad0fe5812939",
-    "text": "Fixed a bug which could prevent a provider ID from ever being set for node if an error occurred determining the provider ID when the node was added.",
-    "markdown": "Fixed a bug which could prevent a provider ID from ever being set for node if an error occurred determining the provider ID when the node was added. ([#87043](https://github.com/kubernetes/kubernetes/pull/87043), [@zjs](https://github.com/zjs))",
-    "author": "zjs",
-    "author_url": "https://github.com/zjs",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87043",
-    "pr_number": 87043,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.15.8"
-  }
-}
diff --git a/src/assets/release-notes-1.15.9.json b/src/assets/release-notes-1.15.9.json
deleted file mode 100644
index 2364960e..00000000
--- a/src/assets/release-notes-1.15.9.json
+++ /dev/null
@@ -1,253 +0,0 @@
-{
-  "76496": {
-    "commit": "f4db8212be53c69a27d893d6a4111422fbce8008",
-    "text": "The following deprecated metrics are removed, please convert to the corresponding metrics:\n1. The following replacement metrics are available from v1.14.0:\n- `rest_client_request_latency_seconds` -\u003e `rest_client_request_duration_seconds`\n- `scheduler_scheduling_latency_seconds` -\u003e `scheduler_scheduling_duration_seconds `\n- `docker_operations` -\u003e `docker_operations_total`\n- `docker_operations_latency_microseconds` -\u003e `docker_operations_duration_seconds`\n- `docker_operations_errors` -\u003e `docker_operations_errors_total`\n- `docker_operations_timeout` -\u003e `docker_operations_timeout_total`\n- `network_plugin_operations_latency_microseconds` -\u003e `network_plugin_operations_duration_seconds`\n- `kubelet_pod_worker_latency_microseconds` -\u003e `kubelet_pod_worker_duration_seconds`\n- `kubelet_pod_start_latency_microseconds` -\u003e `kubelet_pod_start_duration_seconds`\n- `kubelet_cgroup_manager_latency_microseconds` -\u003e `kubelet_cgroup_manager_duration_seconds`\n- `kubelet_pod_worker_start_latency_microseconds` -\u003e `kubelet_pod_worker_start_duration_seconds`\n- `kubelet_pleg_relist_latency_microseconds` -\u003e `kubelet_pleg_relist_duration_seconds`\n- `kubelet_pleg_relist_interval_microseconds` -\u003e `kubelet_pleg_relist_interval_seconds`\n- `kubelet_eviction_stats_age_microseconds` -\u003e `kubelet_eviction_stats_age_seconds`\n- `kubelet_runtime_operations` -\u003e `kubelet_runtime_operations_total`\n- `kubelet_runtime_operations_latency_microseconds` -\u003e `kubelet_runtime_operations_duration_seconds`\n- `kubelet_runtime_operations_errors` -\u003e `kubelet_runtime_operations_errors_total`\n- `kubelet_device_plugin_registration_count` -\u003e `kubelet_device_plugin_registration_total`\n- `kubelet_device_plugin_alloc_latency_microseconds` -\u003e `kubelet_device_plugin_alloc_duration_seconds`\n- `scheduler_e2e_scheduling_latency_microseconds` -\u003e `scheduler_e2e_scheduling_duration_seconds`\n- `scheduler_scheduling_algorithm_latency_microseconds` -\u003e `scheduler_scheduling_algorithm_duration_seconds`\n- `scheduler_scheduling_algorithm_predicate_evaluation` -\u003e `scheduler_scheduling_algorithm_predicate_evaluation_seconds`\n- `scheduler_scheduling_algorithm_priority_evaluation` -\u003e `scheduler_scheduling_algorithm_priority_evaluation_seconds`\n- `scheduler_scheduling_algorithm_preemption_evaluation` -\u003e `scheduler_scheduling_algorithm_preemption_evaluation_seconds`\n- `scheduler_binding_latency_microseconds` -\u003e `scheduler_binding_duration_seconds`\n- `kubeproxy_sync_proxy_rules_latency_microseconds` -\u003e `kubeproxy_sync_proxy_rules_duration_seconds`\n- `apiserver_request_latencies` -\u003e `apiserver_request_duration_seconds`\n- `apiserver_dropped_requests` -\u003e `apiserver_dropped_requests_total`\n- `etcd_request_latencies_summary` -\u003e `etcd_request_duration_seconds`\n- `apiserver_storage_transformation_latencies_microseconds ` -\u003e `apiserver_storage_transformation_duration_seconds`\n- `apiserver_storage_data_key_generation_latencies_microseconds` -\u003e `apiserver_storage_data_key_generation_duration_seconds`\n- `apiserver_request_count` -\u003e `apiserver_request_total`\n- `apiserver_request_latencies_summary`\n2. The following replacement metrics are available from v1.15.0:\n- `apiserver_storage_transformation_failures_total` -\u003e `apiserver_storage_transformation_operations_total`",
-    "markdown": "The following deprecated metrics are removed, please convert to the corresponding metrics:\n  1. The following replacement metrics are available from v1.14.0:\n  - `rest_client_request_latency_seconds` -\u003e `rest_client_request_duration_seconds`\n  - `scheduler_scheduling_latency_seconds` -\u003e `scheduler_scheduling_duration_seconds `\n  - `docker_operations` -\u003e `docker_operations_total`\n  - `docker_operations_latency_microseconds` -\u003e `docker_operations_duration_seconds`\n  - `docker_operations_errors` -\u003e `docker_operations_errors_total`\n  - `docker_operations_timeout` -\u003e `docker_operations_timeout_total`\n  - `network_plugin_operations_latency_microseconds` -\u003e `network_plugin_operations_duration_seconds`\n  - `kubelet_pod_worker_latency_microseconds` -\u003e `kubelet_pod_worker_duration_seconds`\n  - `kubelet_pod_start_latency_microseconds` -\u003e `kubelet_pod_start_duration_seconds`\n  - `kubelet_cgroup_manager_latency_microseconds` -\u003e `kubelet_cgroup_manager_duration_seconds`\n  - `kubelet_pod_worker_start_latency_microseconds` -\u003e `kubelet_pod_worker_start_duration_seconds`\n  - `kubelet_pleg_relist_latency_microseconds` -\u003e `kubelet_pleg_relist_duration_seconds`\n  - `kubelet_pleg_relist_interval_microseconds` -\u003e `kubelet_pleg_relist_interval_seconds`\n  - `kubelet_eviction_stats_age_microseconds` -\u003e `kubelet_eviction_stats_age_seconds`\n  - `kubelet_runtime_operations` -\u003e `kubelet_runtime_operations_total`\n  - `kubelet_runtime_operations_latency_microseconds` -\u003e `kubelet_runtime_operations_duration_seconds`\n  - `kubelet_runtime_operations_errors` -\u003e `kubelet_runtime_operations_errors_total`\n  - `kubelet_device_plugin_registration_count` -\u003e `kubelet_device_plugin_registration_total`\n  - `kubelet_device_plugin_alloc_latency_microseconds` -\u003e `kubelet_device_plugin_alloc_duration_seconds`\n  - `scheduler_e2e_scheduling_latency_microseconds` -\u003e `scheduler_e2e_scheduling_duration_seconds`\n  - `scheduler_scheduling_algorithm_latency_microseconds` -\u003e `scheduler_scheduling_algorithm_duration_seconds`\n  - `scheduler_scheduling_algorithm_predicate_evaluation` -\u003e `scheduler_scheduling_algorithm_predicate_evaluation_seconds`\n  - `scheduler_scheduling_algorithm_priority_evaluation` -\u003e `scheduler_scheduling_algorithm_priority_evaluation_seconds`\n  - `scheduler_scheduling_algorithm_preemption_evaluation` -\u003e `scheduler_scheduling_algorithm_preemption_evaluation_seconds`\n  - `scheduler_binding_latency_microseconds` -\u003e `scheduler_binding_duration_seconds`\n  - `kubeproxy_sync_proxy_rules_latency_microseconds` -\u003e `kubeproxy_sync_proxy_rules_duration_seconds`\n  - `apiserver_request_latencies` -\u003e `apiserver_request_duration_seconds`\n  - `apiserver_dropped_requests` -\u003e `apiserver_dropped_requests_total`\n  - `etcd_request_latencies_summary` -\u003e `etcd_request_duration_seconds`\n  - `apiserver_storage_transformation_latencies_microseconds ` -\u003e `apiserver_storage_transformation_duration_seconds`\n  - `apiserver_storage_data_key_generation_latencies_microseconds` -\u003e `apiserver_storage_data_key_generation_duration_seconds`\n  - `apiserver_request_count` -\u003e `apiserver_request_total`\n  - `apiserver_request_latencies_summary`\n  2. The following replacement metrics are available from v1.15.0:\n  - `apiserver_storage_transformation_failures_total` -\u003e `apiserver_storage_transformation_operations_total` ([#76496](https://github.com/kubernetes/kubernetes/pull/76496), [@danielqsj](https://github.com/danielqsj))",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76496",
-    "pr_number": 76496,
-    "areas": ["apiserver", "ipvs", "kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": [
-      "api-machinery",
-      "cluster-lifecycle",
-      "instrumentation",
-      "network",
-      "node",
-      "scheduling"
-    ],
-    "duplicate": true,
-    "release_version": "1.15.9"
-  },
-  "84265": {
-    "commit": "c9003a268dff6700506929b247847341ea4d5b33",
-    "text": "Fixed the following \n-  AWS Cloud Provider attempts to delete LoadBalancer security group it didn’t provision\n-  AWS Cloud Provider creates default LoadBalancer security group even if annotation [service.beta.kubernetes.io/aws-load-balancer-security-groups] is present",
-    "markdown": "Fixed the following \n  -  AWS Cloud Provider attempts to delete LoadBalancer security group it didn’t provision\n  -  AWS Cloud Provider creates default LoadBalancer security group even if annotation [service.beta.kubernetes.io/aws-load-balancer-security-groups] is present ([#84265](https://github.com/kubernetes/kubernetes/pull/84265), [@bhagwat070919](https://github.com/bhagwat070919))",
-    "author": "bhagwat070919",
-    "author_url": "https://github.com/bhagwat070919",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84265",
-    "pr_number": 84265,
-    "areas": ["cloudprovider", "provider/aws"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.9"
-  },
-  "85281": {
-    "commit": "d9bf22f8f7df184cf427114e8ad1fef12474364f",
-    "text": "New flag `--show-hidden-metrics-for-version` in kube-controller-manager can be used to show all hidden metrics that deprecated in the previous minor release.",
-    "markdown": "New flag `--show-hidden-metrics-for-version` in kube-controller-manager can be used to show all hidden metrics that deprecated in the previous minor release. ([#85281](https://github.com/kubernetes/kubernetes/pull/85281), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG API Machinery",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85281",
-    "pr_number": 85281,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.15.9"
-  },
-  "85283": {
-    "commit": "dd77283426bd3249004472efcbb3257744448161",
-    "text": "Cleaned up the output from `kubectl describe CSINode \u003cname\u003e`.",
-    "markdown": "Cleaned up the output from `kubectl describe CSINode \u003cname\u003e`. ([#85283](https://github.com/kubernetes/kubernetes/pull/85283), [@huffmanca](https://github.com/huffmanca))",
-    "author": "huffmanca",
-    "author_url": "https://github.com/huffmanca",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85283",
-    "pr_number": 85283,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli", "storage"],
-    "duplicate": true,
-    "release_version": "1.15.9"
-  },
-  "85458": {
-    "commit": "7f2128757b1aa2e83320d1f635fe8b2e7028542a",
-    "text": "Improved yaml parsing performance",
-    "markdown": "Improved yaml parsing performance ([#85458](https://github.com/kubernetes/kubernetes/pull/85458), [@cjcullen](https://github.com/cjcullen))",
-    "author": "cjcullen",
-    "author_url": "https://github.com/cjcullen",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85458",
-    "pr_number": 85458,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl"],
-    "kinds": ["bug"],
-    "sigs": [
-      "api-machinery",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "node"
-    ],
-    "duplicate": true,
-    "release_version": "1.15.9"
-  },
-  "85613": {
-    "commit": "05209312b74eac15f0a80577cbbdcb11bce9b8ad",
-    "text": "`kubectl apply  -f \u003cfile\u003e --prune -n \u003cnamespace\u003e` should prune all resources not defined in the file in the cli specified namespace.",
-    "markdown": "`kubectl apply  -f \u003cfile\u003e --prune -n \u003cnamespace\u003e` should prune all resources not defined in the file in the cli specified namespace. ([#85613](https://github.com/kubernetes/kubernetes/pull/85613), [@MartinKaburu](https://github.com/MartinKaburu))",
-    "author": "MartinKaburu",
-    "author_url": "https://github.com/MartinKaburu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85613",
-    "pr_number": 85613,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.15.9"
-  },
-  "85850": {
-    "commit": "61d36e4a43b831f960b190b81d371cb33b5f20d1",
-    "text": "You can now pass \"--node-ip ::\" to kubelet to indicate that it should autodetect an IPv6 address to use as the node's primary address.",
-    "markdown": "You can now pass \"--node-ip ::\" to kubelet to indicate that it should autodetect an IPv6 address to use as the node's primary address. ([#85850](https://github.com/kubernetes/kubernetes/pull/85850), [@danwinship](https://github.com/danwinship))\n\n  Courtesy of SIG Cloud Provider, SIG Network, and SIG Node",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85850",
-    "pr_number": 85850,
-    "areas": ["ipv6", "kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider", "network", "node"],
-    "feature": true,
-    "release_version": "1.15.9"
-  },
-  "86282": {
-    "commit": "be26fbc638ff5ed952ccae70e68baa712b3c60c8",
-    "text": "kubelet: resource metrics endpoint `/metrics/resource/v1alpha1` as well as all metrics under this endpoint have been deprecated.\nPlease convert to the following metrics emitted by endpoint `/metrics/resource`:\n- scrape_error --\u003e scrape_error\n- node_cpu_usage_seconds_total --\u003e node_cpu_usage_seconds\n- node_memory_working_set_bytes --\u003e node_memory_working_set_bytes\n- container_cpu_usage_seconds_total --\u003e container_cpu_usage_seconds\n- container_memory_working_set_bytes --\u003e container_memory_working_set_bytes\n- scrape_error --\u003e scrape_error",
-    "markdown": "kubelet: resource metrics endpoint `/metrics/resource/v1alpha1` as well as all metrics under this endpoint have been deprecated.\n  Please convert to the following metrics emitted by endpoint `/metrics/resource`:\n  - scrape_error --\u003e scrape_error\n  - node_cpu_usage_seconds_total --\u003e node_cpu_usage_seconds\n  - node_memory_working_set_bytes --\u003e node_memory_working_set_bytes\n  - container_cpu_usage_seconds_total --\u003e container_cpu_usage_seconds\n  - container_memory_working_set_bytes --\u003e container_memory_working_set_bytes\n  - scrape_error --\u003e scrape_error ([#86282](https://github.com/kubernetes/kubernetes/pull/86282), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Node",
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86282",
-    "pr_number": 86282,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.15.9"
-  },
-  "86861": {
-    "commit": "916edd922e528f2f27542d3ca5967ca11fe153f8",
-    "text": "kubeadm upgrades always persist the etcd backup for stacked",
-    "markdown": "kubeadm upgrades always persist the etcd backup for stacked ([#86861](https://github.com/kubernetes/kubernetes/pull/86861), [@SataQiu](https://github.com/SataQiu))",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86861",
-    "pr_number": 86861,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.15.9"
-  },
-  "86899": {
-    "commit": "3e8155ead3d890697141d4946ec5fdd5b0dab192",
-    "text": "kubeadm: support automatic retry after failing to pull image",
-    "markdown": "kubeadm: support automatic retry after failing to pull image ([#86899](https://github.com/kubernetes/kubernetes/pull/86899), [@SataQiu](https://github.com/SataQiu))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86899",
-    "pr_number": 86899,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.15.9"
-  },
-  "87029": {
-    "commit": "af46f2cdd2a72ef808f99715c978d58a4603871b",
-    "text": "Fixes service account token admission error in clusters that do not run the service account token controller",
-    "markdown": "Fixes service account token admission error in clusters that do not run the service account token controller ([#87029](https://github.com/kubernetes/kubernetes/pull/87029), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87029",
-    "pr_number": 87029,
-    "kinds": ["bug"],
-    "sigs": ["auth"],
-    "release_version": "1.15.9"
-  },
-  "87035": {
-    "commit": "d2a526bc0eb0961966584749935e7a28bd769adc",
-    "text": "provider/azure: Network security groups can now be in a separate resource group.",
-    "markdown": "provider/azure: Network security groups can now be in a separate resource group. ([#87035](https://github.com/kubernetes/kubernetes/pull/87035), [@CecileRobertMichon](https://github.com/CecileRobertMichon))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "CecileRobertMichon",
-    "author_url": "https://github.com/CecileRobertMichon",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87035",
-    "pr_number": 87035,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.15.9"
-  },
-  "87044": {
-    "commit": "36e40fb850293076b415ae3d376f5f81dc897105",
-    "text": "TODO",
-    "markdown": "TODO ([#87044](https://github.com/kubernetes/kubernetes/pull/87044), [@jennybuckley](https://github.com/jennybuckley))\n\n  Courtesy of SIG API Machinery",
-    "author": "jennybuckley",
-    "author_url": "https://github.com/jennybuckley",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87044",
-    "pr_number": 87044,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.15.9"
-  },
-  "87122": {
-    "commit": "93595eb4eb933bfed90b1f4a1f5597caccf7ccfd",
-    "text": "Azure network and VM clients now suppress requests on throttling",
-    "markdown": "Azure network and VM clients now suppress requests on throttling ([#87122](https://github.com/kubernetes/kubernetes/pull/87122), [@feiskyer](https://github.com/feiskyer))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87122",
-    "pr_number": 87122,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.15.9"
-  },
-  "87203": {
-    "commit": "0c8dcc59f099f69240ffd83eb2eae1cb6e1c68aa",
-    "text": "All apiservers log request lines in a more greppable format.",
-    "markdown": "All apiservers log request lines in a more greppable format. ([#87203](https://github.com/kubernetes/kubernetes/pull/87203), [@lavalamp](https://github.com/lavalamp))",
-    "author": "lavalamp",
-    "author_url": "https://github.com/lavalamp",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87203",
-    "pr_number": 87203,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.15.9"
-  },
-  "87213": {
-    "commit": "a25c6aa35067187800ef2a0928646f5d23268764",
-    "text": "CustomResourceDefinition status fields are no longer required for client validation when submitting manifests.",
-    "markdown": "CustomResourceDefinition status fields are no longer required for client validation when submitting manifests. ([#87213](https://github.com/kubernetes/kubernetes/pull/87213), [@hasheddan](https://github.com/hasheddan))\n\n  Courtesy of SIG API Machinery",
-    "author": "hasheddan",
-    "author_url": "https://github.com/hasheddan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87213",
-    "pr_number": 87213,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.15.9"
-  },
-  "87246": {
-    "commit": "6413f1ee2be99fe5ba18844a2646fb15d462bec4",
-    "text": "Fix the bug PIP's DNS is deleted if no DNS label service annotation isn't set.",
-    "markdown": "Fix the bug PIP's DNS is deleted if no DNS label service annotation isn't set. ([#87246](https://github.com/kubernetes/kubernetes/pull/87246), [@nilo19](https://github.com/nilo19))",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87246",
-    "pr_number": 87246,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.15.9"
-  }
-}
diff --git a/src/assets/release-notes-1.16.0.json b/src/assets/release-notes-1.16.0.json
deleted file mode 100644
index fb851dd7..00000000
--- a/src/assets/release-notes-1.16.0.json
+++ /dev/null
@@ -1,4469 +0,0 @@
-{
-  "81068": {
-    "commit": "e7ecb2240375ac2b91f7bbd55358cded0eb1f698",
-    "text": "",
-    "markdown": "Pod spread constraints have been added in alpha. You can use these constraints to control how Pods are spread across the cluster among failure-domains.",
-    "documentation": [
-      {
-        "description": "Even Pods Spread - 6. Integration Test #80011",
-        "url": "https://github.com/kubernetes/kubernetes/pull/80011",
-        "type": "external"
-      },
-      {
-        "description": "Even Pods Spread - 5. Priority Core #79063",
-        "url": "https://github.com/kubernetes/kubernetes/pull/79063",
-        "type": "external"
-      },
-      {
-        "description": "Even Pods Spread - 4. Preemption Support #79062",
-        "url": "https://github.com/kubernetes/kubernetes/pull/79062",
-        "type": "external"
-      },
-      {
-        "description": "Even Pods Spread - 3. Predicates Core #77828",
-        "url": "https://github.com/kubernetes/kubernetes/pull/77828",
-        "type": "external"
-      },
-      {
-        "description": "Even Pods Spread - 2. Calculating Predicates Metadata #77760",
-        "url": "https://github.com/kubernetes/kubernetes/pull/77760",
-        "type": "external"
-      },
-      {
-        "description": "Even Pods Spread - 1. API changes #77327",
-        "url": "https://github.com/kubernetes/kubernetes/pull/77327",
-        "type": "external"
-      }
-    ],
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81068",
-    "pr_number": 81068,
-    "areas": ["kubeadm", "kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "53185": {
-    "commit": "315dcca3417e6224294322b1073b5e9d03453f26",
-    "text": "populate object name for admission attributes when CREATE",
-    "markdown": "populate object name for admission attributes when CREATE ([#53185](https://github.com/kubernetes/kubernetes/pull/53185), [@dixudx](https://github.com/dixudx))\n\n  Courtesy of SIG API Machinery, SIG Auth, and SIG Testing",
-    "author": "dixudx",
-    "author_url": "https://github.com/dixudx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/53185",
-    "pr_number": 53185,
-    "areas": ["admission-control", "apiserver", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "auth", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "57741": {
-    "commit": "53f4e0fa26c25d7d4eed8fa7f0b3c363ce33f4c8",
-    "text": "Omit nil or empty field when calculating container hash value to avoid hash changed. For a new field with a non-nil default value in the container spec, the hash would still get changed.",
-    "markdown": "Omit nil or empty field when calculating container hash value to avoid hash changed. For a new field with a non-nil default value in the container spec, the hash would still get changed. ([#57741](https://github.com/kubernetes/kubernetes/pull/57741), [@dixudx](https://github.com/dixudx))",
-    "author": "dixudx",
-    "author_url": "https://github.com/dixudx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/57741",
-    "pr_number": 57741,
-    "areas": ["kubelet"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["node"],
-    "release_version": "1.16.0"
-  },
-  "59484": {
-    "commit": "59d29b67d94b3951320c1ee20481bb97425305d0",
-    "text": "Ephemeral containers have been added in alpha. These temporary containers can be added to running pods for purposes such as debugging, similar to how `kubectl exec` runs an process in an existing container. Also like `kubectl exec`, no resources are reserved for ephemeral containers and they are not restarted when they exit.\n\nNote that container namespace targeting is not yet implemented, so [process namespace sharing](https://kubernetes.io/docs/tasks/configure-pod-container/share-process-namespace/) must be enabled to view process from other containers in the pod.",
-    "markdown": "Ephemeral containers have been added in alpha. These temporary containers can be added to running pods for purposes such as debugging, similar to how `kubectl exec` runs an process in an existing container. Also like `kubectl exec`, no resources are reserved for ephemeral containers and they are not restarted when they exit.\n  \n  Note that container namespace targeting is not yet implemented, so [process namespace sharing](https://kubernetes.io/docs/tasks/configure-pod-container/share-process-namespace/) must be enabled to view process from other containers in the pod. ([#59484](https://github.com/kubernetes/kubernetes/pull/59484), [@verb](https://github.com/verb))\n\n  Courtesy of SIG Node",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://git.k8s.io/enhancements/keps/sig-node/20190212-ephemeral-containers.md",
-        "type": "external"
-      }
-    ],
-    "author": "verb",
-    "author_url": "https://github.com/verb",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/59484",
-    "pr_number": 59484,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "64996": {
-    "commit": "d951d19ec4d8ec829e83d146200ea91b664b6fa6",
-    "text": "Property `conditions` in `apiextensions.v1beta1.CustomResourceDefinitionStatus` and `apiextensions.v1.CustomResourceDefinitionStatus` is now optional instead of required.",
-    "markdown": "Property `conditions` in `apiextensions.v1beta1.CustomResourceDefinitionStatus` and `apiextensions.v1.CustomResourceDefinitionStatus` is now optional instead of required. ([#64996](https://github.com/kubernetes/kubernetes/pull/64996), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/64996",
-    "pr_number": 64996,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.0"
-  },
-  "69263": {
-    "commit": "bf55a99eb861262b59375eb3a11d13b39db80031",
-    "text": "Creates an annotation 'service.beta.kubernetes.io/aws-load-balancer-eip-allocations' to assign AWS EIP to the newly created Network Load Balancer. Number of allocations and subnets must match.",
-    "markdown": "Creates an annotation 'service.beta.kubernetes.io/aws-load-balancer-eip-allocations' to assign AWS EIP to the newly created Network Load Balancer. Number of allocations and subnets must match. ([#69263](https://github.com/kubernetes/kubernetes/pull/69263), [@brooksgarrett](https://github.com/brooksgarrett))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "brooksgarrett",
-    "author_url": "https://github.com/brooksgarrett",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/69263",
-    "pr_number": 69263,
-    "areas": ["cloudprovider"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "69655": {
-    "commit": "bcb464db7ba2478429f4deff2c682f4efe46855e",
-    "text": "When the status of a CRD Condition changes, it's corresponding `LastTransitionTime` is now updated.",
-    "markdown": "When the status of a CRD Condition changes, it's corresponding `LastTransitionTime` is now updated. ([#69655](https://github.com/kubernetes/kubernetes/pull/69655), [@CaoShuFeng](https://github.com/CaoShuFeng))",
-    "author": "CaoShuFeng",
-    "author_url": "https://github.com/CaoShuFeng",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/69655",
-    "pr_number": 69655,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.0"
-  },
-  "70672": {
-    "commit": "86e5099b0760a7ef4faf431f1b605bd99177f384",
-    "text": "The following APIs are no longer served by default:\n- All resources under `apps/v1beta1` and `apps/v1beta2` - use `apps/v1` instead\n- `daemonsets`, `deployments`, `replicasets` resources under `extensions/v1beta1` - use `apps/v1` instead\n- `networkpolicies` resources under `extensions/v1beta1` - use `networking.k8s.io/v1` instead\n- `podsecuritypolicies` resources under `extensions/v1beta1` - use `policy/v1beta1` instead\n\nServing these resources can be temporarily re-enabled using the `--runtime-config` apiserver flag. \n- `apps/v1beta1=true`\n- `apps/v1beta2=true`\n- `extensions/v1beta1/daemonsets=true,extensions/v1beta1/deployments=true,extensions/v1beta1/replicasets=true,extensions/v1beta1/networkpolicies=true,extensions/v1beta1/podsecuritypolicies=true`\n\nThe ability to serve these resources will be completely removed in v1.18.",
-    "markdown": "The following APIs are no longer served by default:\n  - All resources under `apps/v1beta1` and `apps/v1beta2` - use `apps/v1` instead\n  - `daemonsets`, `deployments`, `replicasets` resources under `extensions/v1beta1` - use `apps/v1` instead\n  - `networkpolicies` resources under `extensions/v1beta1` - use `networking.k8s.io/v1` instead\n  - `podsecuritypolicies` resources under `extensions/v1beta1` - use `policy/v1beta1` instead\n  \n  Serving these resources can be temporarily re-enabled using the `--runtime-config` apiserver flag. \n  - `apps/v1beta1=true`\n  - `apps/v1beta2=true`\n  - `extensions/v1beta1/daemonsets=true,extensions/v1beta1/deployments=true,extensions/v1beta1/replicasets=true,extensions/v1beta1/networkpolicies=true,extensions/v1beta1/podsecuritypolicies=true`\n  \n  The ability to serve these resources will be completely removed in v1.18. ([#70672](https://github.com/kubernetes/kubernetes/pull/70672), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Network, and SIG Testing",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/70672",
-    "pr_number": 70672,
-    "areas": ["apiserver", "test"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["api-machinery", "apps", "network", "testing"],
-    "action_required": true,
-    "release_version": "1.16.0"
-  },
-  "71757": {
-    "commit": "bc1d8c6d61078269540f807248a455cba09e672c",
-    "text": "Implement request cancellation in token transport to get rid of warnings when the HTTP client tries to cancel a request.",
-    "markdown": "Implement request cancellation in token transport to get rid of warnings when the HTTP client tries to cancel a request. ([#71757](https://github.com/kubernetes/kubernetes/pull/71757), [@mikedanese](https://github.com/mikedanese))",
-    "author": "mikedanese",
-    "author_url": "https://github.com/mikedanese",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/71757",
-    "pr_number": 71757,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "72416": {
-    "commit": "5625eadd2a3b73fc6bd328bc10ec238c7ab84e0f",
-    "text": "`kubectl get -w` now takes an `--output-watch-events` flag to indicate the event type (ADDED, MODIFIED, DELETED)",
-    "markdown": "`kubectl get -w` now takes an `--output-watch-events` flag to indicate the event type (ADDED, MODIFIED, DELETED) ([#72416](https://github.com/kubernetes/kubernetes/pull/72416), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG CLI, and SIG Testing",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72416",
-    "pr_number": 72416,
-    "areas": ["kubectl", "test"],
-    "kinds": ["feature"],
-    "sigs": ["cli", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "72420": {
-    "commit": "2a82853614b710d30c334d41723661beb1ffcfd3",
-    "text": "The deprecated node condition type 'OutOfDisk' has been removed. Use the 'DiskPressure' condition instead.",
-    "markdown": "The deprecated node condition type 'OutOfDisk' has been removed. Use the 'DiskPressure' condition instead. ([#72420](https://github.com/kubernetes/kubernetes/pull/72420), [@Pingan2017](https://github.com/Pingan2017))",
-    "author": "Pingan2017",
-    "author_url": "https://github.com/Pingan2017",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72420",
-    "pr_number": 72420,
-    "areas": ["kubelet"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["api-machinery", "apps", "architecture", "node", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "73405": {
-    "commit": "efaacf786a3d04a9ee3fdc167fc2162221d95dfd",
-    "text": "Add status condition to namespace resource",
-    "markdown": "Add status condition to namespace resource ([#73405](https://github.com/kubernetes/kubernetes/pull/73405), [@wozniakjan](https://github.com/wozniakjan))\n\n  Courtesy of SIG API Machinery, SIG Apps, and SIG Architecture",
-    "author": "wozniakjan",
-    "author_url": "https://github.com/wozniakjan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73405",
-    "pr_number": 73405,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "architecture"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "73923": {
-    "commit": "98c68df6069dc8f81c1f983609be60b2045861cb",
-    "text": "On Windows systems, %USERPROFILE% is now preferred over %HOMEDRIVE%\\%HOMEPATH% as the home folder if %HOMEDRIVE%\\%HOMEPATH% does not contain a .kube\\config file, and %USERPROFILE% exists and is writeable.",
-    "markdown": "On Windows systems, %USERPROFILE% is now preferred over %HOMEDRIVE%\\%HOMEPATH% as the home folder if %HOMEDRIVE%\\%HOMEPATH% does not contain a .kube\\config file, and %USERPROFILE% exists and is writeable. ([#73923](https://github.com/kubernetes/kubernetes/pull/73923), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery, SIG CLI, and SIG Windows",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73923",
-    "pr_number": 73923,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cli", "windows"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "73977": {
-    "commit": "a33840e0232aaf2b5c14c1c754d9189d7e841ec1",
-    "text": "To configure controller manager to use ipv6dual stack:\nuse --cluster-cidr=\"\u003ccidr1\u003e,\u003ccidr2\u003e\".\n\nNotes:\n \n1. Only the first two cidrs are used (soft limits for Alpha, might be lifted later on). \n2. Only the \"RangeAllocator\" (default) is allowed as a value for --cidr-allocator-type . Cloud allocators are not compatible with ipv6dualstack",
-    "markdown": "To configure controller manager to use ipv6dual stack:\n  use --cluster-cidr=\"\u003ccidr1\u003e,\u003ccidr2\u003e\".\n  \n  Notes:\n   \n  1. Only the first two cidrs are used (soft limits for Alpha, might be lifted later on). \n  2. Only the \"RangeAllocator\" (default) is allowed as a value for --cidr-allocator-type . Cloud allocators are not compatible with ipv6dualstack ([#73977](https://github.com/kubernetes/kubernetes/pull/73977), [@khenidak](https://github.com/khenidak))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Architecture, SIG CLI, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Network, SIG Node, SIG Release, SIG Scheduling, and SIG Testing",
-    "author": "khenidak",
-    "author_url": "https://github.com/khenidak",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73977",
-    "pr_number": 73977,
-    "areas": [
-      "apiserver",
-      "cloudprovider",
-      "code-generation",
-      "dependency",
-      "kubectl",
-      "kubelet",
-      "release-eng",
-      "test"
-    ],
-    "kinds": ["api-change", "feature"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "architecture",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "network",
-      "node",
-      "release",
-      "scheduling",
-      "testing"
-    ],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "73982": {
-    "commit": "75a39b70072954634169f7bc0731795019d04bac",
-    "text": "Fixes invalid \"time stamp is the future\" error when kubectl cp-ing a file",
-    "markdown": "Fixes invalid \"time stamp is the future\" error when kubectl cp-ing a file ([#73982](https://github.com/kubernetes/kubernetes/pull/73982), [@tanshanshan](https://github.com/tanshanshan))",
-    "author": "tanshanshan",
-    "author_url": "https://github.com/tanshanshan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73982",
-    "pr_number": 73982,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.16.0"
-  },
-  "74416": {
-    "commit": "7e17aeb1b5c817e37fbb36278004c63011be3544",
-    "text": "Add --shutdown-delay-duration to kube-apiserver in order to delay a graceful shutdown. `/healthz` will keep returning success during this time and requests are normally served, but `/readyz` will return faillure immediately. This delay can be used to allow the SDN to update iptables on all nodes and stop sending traffic.",
-    "markdown": "Add --shutdown-delay-duration to kube-apiserver in order to delay a graceful shutdown. `/healthz` will keep returning success during this time and requests are normally served, but `/readyz` will return faillure immediately. This delay can be used to allow the SDN to update iptables on all nodes and stop sending traffic. ([#74416](https://github.com/kubernetes/kubernetes/pull/74416), [@sttts](https://github.com/sttts))\n\n  Courtesy of SIG API Machinery",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74416",
-    "pr_number": 74416,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "74526": {
-    "commit": "5ece88c4c81ce40c0f248797f0951564d5bc7bf3",
-    "text": "When HPAScaleToZero feature gate is enabled HPA supports scaling to zero pods based on object or external metrics. HPA remains active as long as at least one metric value available.\n\nTo downgrade the cluster to version that does not support scale-to-zero feature:\n1. make sure there are no hpa objects with minReplicas=0. Here is a oneliner to update it to 1:\n    $ kubectl get hpa --all-namespaces  --no-headers=true | awk  '{if($6==0) printf \"kubectl patch hpa/%s --namespace=%s -p \\\"{\\\\\\\"spec\\\\\\\":{\\\\\\\"minReplicas\\\\\\\":1}}\\\"\\n\", $2, $1 }' | sh\n2. disable HPAScaleToZero feature gate",
-    "markdown": "When HPAScaleToZero feature gate is enabled HPA supports scaling to zero pods based on object or external metrics. HPA remains active as long as at least one metric value available.\n  \n  To downgrade the cluster to version that does not support scale-to-zero feature:\n  1. make sure there are no hpa objects with minReplicas=0. Here is a oneliner to update it to 1:\n      $ kubectl get hpa --all-namespaces  --no-headers=true | awk  '{if($6==0) printf \"kubectl patch hpa/%s --namespace=%s -p \\\"{\\\\\\\"spec\\\\\\\":{\\\\\\\"minReplicas\\\\\\\":1}}\\\"\\n\", $2, $1 }' | sh\n  2. disable HPAScaleToZero feature gate ([#74526](https://github.com/kubernetes/kubernetes/pull/74526), [@DXist](https://github.com/DXist))\n\n  Courtesy of SIG API Machinery, SIG Apps, and SIG Autoscaling",
-    "author": "DXist",
-    "author_url": "https://github.com/DXist",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74526",
-    "pr_number": 74526,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "autoscaling"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "74691": {
-    "commit": "514a157adff2dddf5928bdd8da3288e02ec9de49",
-    "text": "Use O_CLOEXEC to ensure file descriptors do not leak to subprocesses.",
-    "markdown": "Use O_CLOEXEC to ensure file descriptors do not leak to subprocesses. ([#74691](https://github.com/kubernetes/kubernetes/pull/74691), [@cpuguy83](https://github.com/cpuguy83))",
-    "author": "cpuguy83",
-    "author_url": "https://github.com/cpuguy83",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74691",
-    "pr_number": 74691,
-    "areas": ["dependency", "kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "storage"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "74943": {
-    "commit": "bf86300241e51f011040d208e43f1cd2a5c0378a",
-    "text": "kubectl logs --selector will support --tail=-1.",
-    "markdown": "kubectl logs --selector will support --tail=-1. ([#74943](https://github.com/kubernetes/kubernetes/pull/74943), [@JishanXing](https://github.com/JishanXing))",
-    "author": "JishanXing",
-    "author_url": "https://github.com/JishanXing",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74943",
-    "pr_number": 74943,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "cli"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "75223": {
-    "commit": "cff6e420056f65469a30460ecc45111c3b71a799",
-    "text": "Init container resource requests now impact pod QoS class",
-    "markdown": "Init container resource requests now impact pod QoS class ([#75223](https://github.com/kubernetes/kubernetes/pull/75223), [@sjenning](https://github.com/sjenning))",
-    "author": "sjenning",
-    "author_url": "https://github.com/sjenning",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75223",
-    "pr_number": 75223,
-    "kinds": ["bug"],
-    "sigs": ["apps", "node"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "75229": {
-    "commit": "e8559f7565863f3e852ec24cbd9f8322a80e5f30",
-    "text": "Attempt to set the kubelet's hostname \u0026 internal IP if `--cloud-provider=external` and no node addresses exists",
-    "markdown": "Attempt to set the kubelet's hostname \u0026 internal IP if `--cloud-provider=external` and no node addresses exists ([#75229](https://github.com/kubernetes/kubernetes/pull/75229), [@andrewsykim](https://github.com/andrewsykim))\n\n  Courtesy of SIG Node",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75229",
-    "pr_number": 75229,
-    "areas": ["cloudprovider", "kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "75593": {
-    "commit": "a61006bbc4af6f2d2fe1b254133ff7be4cb27476",
-    "text": "E2E tests no longer add command line flags directly to the command line, test suites that want that need to be updated if they don't use HandleFlags\nloading a -viper-config=e2e.yaml with suffix (introduced in 1.13) works again and now has a regression test",
-    "markdown": "E2E tests no longer add command line flags directly to the command line, test suites that want that need to be updated if they don't use HandleFlags\n  loading a -viper-config=e2e.yaml with suffix (introduced in 1.13) works again and now has a regression test ([#75593](https://github.com/kubernetes/kubernetes/pull/75593), [@pohly](https://github.com/pohly))",
-    "author": "pohly",
-    "author_url": "https://github.com/pohly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75593",
-    "pr_number": 75593,
-    "areas": ["e2e-test-framework", "kubeadm", "kubectl", "test"],
-    "kinds": ["bug"],
-    "sigs": ["cli", "cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "75805": {
-    "commit": "748849bbb92170c2acd6fe2a96c95a1362e7129a",
-    "text": "NFS Drivers are now enabled to collect metrics, StatFS metrics provider is used to collect the metrics. (\u0026#35;75805 , @brahmaroutu)",
-    "markdown": "NFS Drivers are now enabled to collect metrics, StatFS metrics provider is used to collect the metrics. (\u0026#35;75805 , @brahmaroutu) ([#75805](https://github.com/kubernetes/kubernetes/pull/75805), [@brahmaroutu](https://github.com/brahmaroutu))",
-    "author": "brahmaroutu",
-    "author_url": "https://github.com/brahmaroutu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75805",
-    "pr_number": 75805,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.16.0"
-  },
-  "76124": {
-    "commit": "1a80962db6fc7138f4d2925a82d7e74cd5ec4fcd",
-    "text": "Fix CRD validation error on 'items' field.",
-    "markdown": "Fix CRD validation error on 'items' field. ([#76124](https://github.com/kubernetes/kubernetes/pull/76124), [@tossmilestone](https://github.com/tossmilestone))",
-    "author": "tossmilestone",
-    "author_url": "https://github.com/tossmilestone",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76124",
-    "pr_number": 76124,
-    "areas": [
-      "apiserver",
-      "cloudprovider",
-      "code-generation",
-      "custom-resources",
-      "dependency",
-      "kubectl"
-    ],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "node"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "76463": {
-    "commit": "9dad928e6ba1d7cf8b67a9f401775d2c8b71a2d0",
-    "text": "Non nil DataSource entries on PVC's are now displayed as part of `describe pvc` output.",
-    "markdown": "Non nil DataSource entries on PVC's are now displayed as part of `describe pvc` output. ([#76463](https://github.com/kubernetes/kubernetes/pull/76463), [@j-griffith](https://github.com/j-griffith))",
-    "author": "j-griffith",
-    "author_url": "https://github.com/j-griffith",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76463",
-    "pr_number": 76463,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli", "storage"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "76518": {
-    "commit": "920ac08361a0c96f2379bdeb3414003796be7856",
-    "text": "Limit the response body length of http readiness/liveness probes to 10KB;",
-    "markdown": "Limit the response body length of http readiness/liveness probes to 10KB; ([#76518](https://github.com/kubernetes/kubernetes/pull/76518), [@haiyanmeng](https://github.com/haiyanmeng))",
-    "author": "haiyanmeng",
-    "author_url": "https://github.com/haiyanmeng",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76518",
-    "pr_number": 76518,
-    "areas": ["apiserver", "cloudprovider", "dependency", "kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "network", "node"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "76968": {
-    "commit": "155f4a649b39bbc5627993537fc280ae43c13a75",
-    "text": "Addition of Overhead field to the PodSpec and RuntimeClass types as part of the Pod Overhead KEP",
-    "markdown": "Addition of Overhead field to the PodSpec and RuntimeClass types as part of the Pod Overhead KEP ([#76968](https://github.com/kubernetes/kubernetes/pull/76968), [@egernst](https://github.com/egernst))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG CLI, and SIG Node",
-    "author": "egernst",
-    "author_url": "https://github.com/egernst",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76968",
-    "pr_number": 76968,
-    "areas": ["kubectl", "kubelet"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "cli", "node"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "77167": {
-    "commit": "c8c9d2e21ce9afe52ac8ace64f96a469891dc724",
-    "text": "Improvement in Kube-proxy. Kube-proxy waits for some duration for the node to be defined.",
-    "markdown": "Improvement in Kube-proxy. Kube-proxy waits for some duration for the node to be defined. ([#77167](https://github.com/kubernetes/kubernetes/pull/77167), [@paulsubrata55](https://github.com/paulsubrata55))",
-    "author": "paulsubrata55",
-    "author_url": "https://github.com/paulsubrata55",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77167",
-    "pr_number": 77167,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.16.0"
-  },
-  "77327": {
-    "commit": "323356fda06cbac0b74c35f522f17c3bb9d580ca",
-    "text": "`[]TopologySpreadConstraint` is introduced into PodSpec to support the \"Even Pods Spread\" alpha feature.",
-    "markdown": "`[]TopologySpreadConstraint` is introduced into PodSpec to support the \"Even Pods Spread\" alpha feature. ([#77327](https://github.com/kubernetes/kubernetes/pull/77327), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77327",
-    "pr_number": 77327,
-    "areas": ["kubeadm", "kubelet", "test"],
-    "kinds": ["api-change"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "cluster-lifecycle",
-      "node",
-      "scheduling",
-      "storage",
-      "testing"
-    ],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "77354": {
-    "commit": "ab162cd28c332d0ecfb4f918d5f91e9e57acdb61",
-    "text": "Server-side apply will now use the openapi provided in the CRD validation field to help figure out how to correctly merge objects and update ownership.",
-    "markdown": "Server-side apply will now use the openapi provided in the CRD validation field to help figure out how to correctly merge objects and update ownership. ([#77354](https://github.com/kubernetes/kubernetes/pull/77354), [@jennybuckley](https://github.com/jennybuckley))\n\n  Courtesy of SIG API Machinery, SIG CLI, SIG Cloud Provider, SIG Cluster Lifecycle, and SIG Testing",
-    "author": "jennybuckley",
-    "author_url": "https://github.com/jennybuckley",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77354",
-    "pr_number": 77354,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "77355": {
-    "commit": "a172e197b165f60c215b4467175bdd61d8a15e0e",
-    "text": "Update gogo/protobuf to serialize backward, as to get better performance on deep objects.",
-    "markdown": "Update gogo/protobuf to serialize backward, as to get better performance on deep objects. ([#77355](https://github.com/kubernetes/kubernetes/pull/77355), [@apelisse](https://github.com/apelisse))",
-    "author": "apelisse",
-    "author_url": "https://github.com/apelisse",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77355",
-    "pr_number": 77355,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl", "kubelet"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery", "auth", "cli", "cloud-provider", "cluster-lifecycle", "node"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "77449": {
-    "commit": "7c7d70bc7c10f774e1bda021de8f337d075a5563",
-    "text": "Kubernetes now supports transparent compression of API responses. Clients that send `Accept-Encoding: gzip` will now receive a GZIP compressed response body if the API call was larger than 128KB.  Go clients automatically request gzip-encoding by default and should see reduced transfer times for very large API requests.  Clients in other languages may need to make changes to benefit from compression.",
-    "markdown": "Kubernetes now supports transparent compression of API responses. Clients that send `Accept-Encoding: gzip` will now receive a GZIP compressed response body if the API call was larger than 128KB.  Go clients automatically request gzip-encoding by default and should see reduced transfer times for very large API requests.  Clients in other languages may need to make changes to benefit from compression. ([#77449](https://github.com/kubernetes/kubernetes/pull/77449), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77449",
-    "pr_number": 77449,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery", "scalability"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "77561": {
-    "commit": "3be827e912d1db8570f653a05447e6e78a3a844b",
-    "text": "Use HTTPS as etcd-apiserver protocol when mTLS between etcd and kube-apiserver on master is enabled, change etcd metrics/health port to 2382.",
-    "markdown": "Use HTTPS as etcd-apiserver protocol when mTLS between etcd and kube-apiserver on master is enabled, change etcd metrics/health port to 2382. ([#77561](https://github.com/kubernetes/kubernetes/pull/77561), [@wenjiaswe](https://github.com/wenjiaswe))",
-    "author": "wenjiaswe",
-    "author_url": "https://github.com/wenjiaswe",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77561",
-    "pr_number": 77561,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cluster-lifecycle"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "77595": {
-    "commit": "22fb6fd174660e5e0e46b9af41482efcfb057f7e",
-    "text": "Integrated volume limits for in-tree and CSI volumes into one scheduler predicate.",
-    "markdown": "Integrated volume limits for in-tree and CSI volumes into one scheduler predicate. ([#77595](https://github.com/kubernetes/kubernetes/pull/77595), [@bertinatto](https://github.com/bertinatto))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Instrumentation, SIG Node, SIG Release, SIG Scheduling, SIG Storage, and SIG Testing",
-    "author": "bertinatto",
-    "author_url": "https://github.com/bertinatto",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77595",
-    "pr_number": 77595,
-    "areas": ["cloudprovider", "e2e-test-framework", "kubeadm", "kubelet", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "auth",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "node",
-      "release",
-      "scheduling",
-      "storage",
-      "testing"
-    ],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "77688": {
-    "commit": "668bf42d11dd61a93e606ba0a1c9b9b6acbed2d3",
-    "text": "Updates the requestedToCapacityRatioArguments to add resources parameter that allows the users to specify the resource name along with weights for each resource to score nodes based on the request to capacity ratio.",
-    "markdown": "Updates the requestedToCapacityRatioArguments to add resources parameter that allows the users to specify the resource name along with weights for each resource to score nodes based on the request to capacity ratio. ([#77688](https://github.com/kubernetes/kubernetes/pull/77688), [@sudeshsh](https://github.com/sudeshsh))\n\n  Courtesy of SIG Scheduling",
-    "author": "sudeshsh",
-    "author_url": "https://github.com/sudeshsh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77688",
-    "pr_number": 77688,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "77712": {
-    "commit": "4b8a1163aa5193e32164dab267d867ab4d085ee1",
-    "text": "if targetPort is changed that will process by service controller",
-    "markdown": "if targetPort is changed that will process by service controller ([#77712](https://github.com/kubernetes/kubernetes/pull/77712), [@Sn0rt](https://github.com/Sn0rt))\n\n  Courtesy of SIG Apps, and SIG Network",
-    "author": "Sn0rt",
-    "author_url": "https://github.com/Sn0rt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77712",
-    "pr_number": 77712,
-    "kinds": ["feature"],
-    "sigs": ["apps", "network"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "77792": {
-    "commit": "003c4e51555f6e9fce8b62ba6916d5b70b2f3010",
-    "text": "kubeadm: prevent PSP blocking of upgrade image prepull by using a non-root user",
-    "markdown": "kubeadm: prevent PSP blocking of upgrade image prepull by using a non-root user ([#77792](https://github.com/kubernetes/kubernetes/pull/77792), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77792",
-    "pr_number": 77792,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "77807": {
-    "commit": "3ebe6a6a5fca72759a0482907238221ea3be5c82",
-    "text": "Added new startupProbe, related to KEP https://github.com/kubernetes/enhancements/issues/950.",
-    "markdown": "Added new startupProbe, related to KEP https://github.com/kubernetes/enhancements/issues/950. ([#77807](https://github.com/kubernetes/kubernetes/pull/77807), [@matthyx](https://github.com/matthyx))",
-    "author": "matthyx",
-    "author_url": "https://github.com/matthyx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77807",
-    "pr_number": 77807,
-    "areas": ["dependency", "kubectl", "kubelet", "release-eng", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["apps", "cli", "node", "release", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "77819": {
-    "commit": "024c7bd84f1e9a386778e14068f57c3b1e193565",
-    "text": "A new client `k8s.io/client-go/metadata.Client` has been added for accessing objects generically. This client makes it easier to retrieve only the metadata (the `metadata` sub-section) from resources on the cluster in an efficient manner for use cases that deal with objects generically, like the garbage collector, quota, or the namespace controller. The client asks the server to return a `meta.k8s.io/v1 PartialObjectMetadata` object for list, get, delete, watch, and patch operations on both normal APIs and custom resources which can be encoded in protobuf for additional work. If the server does not yet support this API the client will gracefully fall back to JSON and transform the response objects into PartialObjectMetadata.",
-    "markdown": "A new client `k8s.io/client-go/metadata.Client` has been added for accessing objects generically. This client makes it easier to retrieve only the metadata (the `metadata` sub-section) from resources on the cluster in an efficient manner for use cases that deal with objects generically, like the garbage collector, quota, or the namespace controller. The client asks the server to return a `meta.k8s.io/v1 PartialObjectMetadata` object for list, get, delete, watch, and patch operations on both normal APIs and custom resources which can be encoded in protobuf for additional work. If the server does not yet support this API the client will gracefully fall back to JSON and transform the response objects into PartialObjectMetadata. ([#77819](https://github.com/kubernetes/kubernetes/pull/77819), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77819",
-    "pr_number": 77819,
-    "areas": ["apiserver", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "77824": {
-    "commit": "e2f57be0c029cf80451983b8aaf49ed476115f6d",
-    "text": "Audit events now log the existence and patch of mutating webhooks. \n- At Metadata audit level or higher, an annotation with key \"mutation.webhook.admission.k8s.io/round_{round idx}_index_{order idx}\" gets logged with JSON payload indicating a webhook gets invoked for given request and whether it mutated the object or not.\n- At Request audit level or higher, an annotation with key \"patch.webhook.admission.k8s.io/round_{round idx}_index_{order idx}\" get logged with the JSON payload logging the patch sent by a webhook for given request.",
-    "markdown": "Audit events now log the existence and patch of mutating webhooks. \n  - At Metadata audit level or higher, an annotation with key \"mutation.webhook.admission.k8s.io/round_{round idx}_index_{order idx}\" gets logged with JSON payload indicating a webhook gets invoked for given request and whether it mutated the object or not.\n  - At Request audit level or higher, an annotation with key \"patch.webhook.admission.k8s.io/round_{round idx}_index_{order idx}\" get logged with the JSON payload logging the patch sent by a webhook for given request. ([#77824](https://github.com/kubernetes/kubernetes/pull/77824), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/77824",
-    "pr_number": 77824,
-    "areas": ["apiserver", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "auth", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "78033": {
-    "commit": "e1c2c677f14b41fff3bb1752e08ec7cdfd535aa8",
-    "text": "Kubeadm now seamlessly migrates the CoreDNS Configuration when upgrading CoreDNS.",
-    "markdown": "Kubeadm now seamlessly migrates the CoreDNS Configuration when upgrading CoreDNS. ([#78033](https://github.com/kubernetes/kubernetes/pull/78033), [@rajansandeep](https://github.com/rajansandeep))\n\n  Courtesy of SIG API Machinery, SIG CLI, and SIG Cluster Lifecycle",
-    "author": "rajansandeep",
-    "author_url": "https://github.com/rajansandeep",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78033",
-    "pr_number": 78033,
-    "areas": ["dependency", "kubeadm", "kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cli", "cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "78097": {
-    "commit": "cc270c138da473430d7243e1fbe4fbadf03cc49c",
-    "text": "Implement \"post-filter\" extension point for scheduling framework",
-    "markdown": "Implement \"post-filter\" extension point for scheduling framework ([#78097](https://github.com/kubernetes/kubernetes/pull/78097), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78097",
-    "pr_number": 78097,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "78098": {
-    "commit": "3cf2d32f871c73c43bd3e477583373735a368976",
-    "text": "Remove kubectl log, use kubectl logs instead",
-    "markdown": "Remove kubectl log, use kubectl logs instead ([#78098](https://github.com/kubernetes/kubernetes/pull/78098), [@soltysh](https://github.com/soltysh))",
-    "author": "soltysh",
-    "author_url": "https://github.com/soltysh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78098",
-    "pr_number": 78098,
-    "areas": ["kubectl", "test"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "78140": {
-    "commit": "416a717effeb3b0ff69e3363b15a9f6570a1a4a1",
-    "text": "Optimize EC2 DescribeInstances API calls in aws cloud provider library by querying instance ID instead of EC2 filters when possible",
-    "markdown": "Optimize EC2 DescribeInstances API calls in aws cloud provider library by querying instance ID instead of EC2 filters when possible ([#78140](https://github.com/kubernetes/kubernetes/pull/78140), [@zhan849](https://github.com/zhan849))\n\n  Courtesy of SIG Cloud Provider, and SIG Scalability",
-    "author": "zhan849",
-    "author_url": "https://github.com/zhan849",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78140",
-    "pr_number": 78140,
-    "areas": ["cloudprovider"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider", "scalability"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "78162": {
-    "commit": "c30f0248649c15a46ab99eb722de0448988198f8",
-    "text": "Add support for writing out of tree custom scheduler plugins.",
-    "markdown": "Add support for writing out of tree custom scheduler plugins. ([#78162](https://github.com/kubernetes/kubernetes/pull/78162), [@hex108](https://github.com/hex108))\n\n  Courtesy of SIG Scheduling",
-    "author": "hex108",
-    "author_url": "https://github.com/hex108",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78162",
-    "pr_number": 78162,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "78170": {
-    "commit": "bbdd0557b69e07e0cd1bf4377f725cafe9888b0d",
-    "text": "Fix a bug that causes DaemonSet rolling update hang when there exist failed pods.",
-    "markdown": "Fix a bug that causes DaemonSet rolling update hang when there exist failed pods. ([#78170](https://github.com/kubernetes/kubernetes/pull/78170), [@DaiHao](https://github.com/DaiHao))",
-    "author": "DaiHao",
-    "author_url": "https://github.com/DaiHao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78170",
-    "pr_number": 78170,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.16.0"
-  },
-  "78294": {
-    "commit": "eee3e976d871bcbf97821a0fa9c129a4c568ac5e",
-    "text": "Removed deprecated flag `--resource-container` from kube-proxy.\n\nThe deprecated `--resource-container` flag has been removed from kube-proxy, and specifying it will now cause an error.  The behavior is now as if you specified `--resource-container=\"\"`.  If you previously specified a non-empty `--resource-container`, you can no longer do so as of kubernetes 1.16.",
-    "markdown": "Removed deprecated flag `--resource-container` from kube-proxy.\n  \n  The deprecated `--resource-container` flag has been removed from kube-proxy, and specifying it will now cause an error.  The behavior is now as if you specified `--resource-container=\"\"`.  If you previously specified a non-empty `--resource-container`, you can no longer do so as of kubernetes 1.16. ([#78294](https://github.com/kubernetes/kubernetes/pull/78294), [@vllry](https://github.com/vllry))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Network",
-    "author": "vllry",
-    "author_url": "https://github.com/vllry",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78294",
-    "pr_number": 78294,
-    "areas": ["kubeadm"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "action_required": true,
-    "release_version": "1.16.0"
-  },
-  "78319": {
-    "commit": "ce80aeaac595e81361c051fe6a9a951ebe14d569",
-    "text": "Introduction of the pod overhead feature to the scheduler.  This functionality is alpha-level as of \nKubernetes v1.16, and is only honored by servers that enable the PodOverhead feature.gate.",
-    "markdown": "Introduction of the pod overhead feature to the scheduler.  This functionality is alpha-level as of \n  Kubernetes v1.16, and is only honored by servers that enable the PodOverhead feature.gate. ([#78319](https://github.com/kubernetes/kubernetes/pull/78319), [@egernst](https://github.com/egernst))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Node, and SIG Scheduling",
-    "author": "egernst",
-    "author_url": "https://github.com/egernst",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78319",
-    "pr_number": 78319,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "node", "scheduling"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "78447": {
-    "commit": "b0d08f6a47c7d3557e220634bcdb900398b4034b",
-    "text": "Migrate scheduler to use v1beta1 Event API. any tool targeting scheduler events needs to use v1beta1 Event API",
-    "markdown": "Migrate scheduler to use v1beta1 Event API. any tool targeting scheduler events needs to use v1beta1 Event API ([#78447](https://github.com/kubernetes/kubernetes/pull/78447), [@yastij](https://github.com/yastij))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG Scheduling, and SIG Testing",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78447",
-    "pr_number": 78447,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "apps", "auth", "scheduling", "testing"],
-    "feature": true,
-    "action_required": true,
-    "release_version": "1.16.0"
-  },
-  "78458": {
-    "commit": "ae3c44d043769305c6add665cd81c16319479ba7",
-    "text": "The kube-apiserver has improved behavior for both startup and shutdown sequences and also now exposes `\\readyz` for readiness checking. Readyz includes all existing healthz checks but also adds a shutdown check. When a cluster admin initiates a shutdown, the kube-apiserver will try to process existing requests (for the duration of request timeout) before killing the apiserver process.  \n\nThe apiserver also now takes an optional flag \"--maximum-startup-sequence-duration\". This allows you to explicitly define an upper bound on the apiserver startup sequences before healthz begins to fail. By keeping the kubelet liveness initial delay short, this can enable quick kubelet recovery as soon as we have a boot sequence which has not completed in our expected time frame, despite lack of completion from longer boot sequences (like RBAC). Kube-apiserver behavior when the value of this flag is zero is backwards compatible (this is as the defaulted value of the flag).",
-    "markdown": "The kube-apiserver has improved behavior for both startup and shutdown sequences and also now exposes `\\readyz` for readiness checking. Readyz includes all existing healthz checks but also adds a shutdown check. When a cluster admin initiates a shutdown, the kube-apiserver will try to process existing requests (for the duration of request timeout) before killing the apiserver process.  \n  \n  The apiserver also now takes an optional flag \"--maximum-startup-sequence-duration\". This allows you to explicitly define an upper bound on the apiserver startup sequences before healthz begins to fail. By keeping the kubelet liveness initial delay short, this can enable quick kubelet recovery as soon as we have a boot sequence which has not completed in our expected time frame, despite lack of completion from longer boot sequences (like RBAC). Kube-apiserver behavior when the value of this flag is zero is backwards compatible (this is as the defaulted value of the flag). ([#78458](https://github.com/kubernetes/kubernetes/pull/78458), [@logicalhan](https://github.com/logicalhan))",
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78458",
-    "pr_number": 78458,
-    "areas": ["apiserver", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "auth", "cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "78477": {
-    "commit": "f31d786927a7855b3dd2200d16e2b172661fb919",
-    "text": "Add Filter extension point to the scheduling framework.",
-    "markdown": "Add Filter extension point to the scheduling framework. ([#78477](https://github.com/kubernetes/kubernetes/pull/78477), [@YoubingLi](https://github.com/YoubingLi))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "YoubingLi",
-    "author_url": "https://github.com/YoubingLi",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78477",
-    "pr_number": 78477,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "78484": {
-    "commit": "e4f15883522673a57349641f7601a816470a98b6",
-    "text": "Introduce a new admission controller for RuntimeClass. Initially, RuntimeClass will be used to apply the pod overhead associated with a given RuntimeClass to the Pod.Spec if a corresponding RuntimeClassName is specified.\n\nPodOverhead is an alpha feature as of Kubernetes 1.16.",
-    "markdown": "Introduce a new admission controller for RuntimeClass. Initially, RuntimeClass will be used to apply the pod overhead associated with a given RuntimeClass to the Pod.Spec if a corresponding RuntimeClassName is specified.\n  \n  PodOverhead is an alpha feature as of Kubernetes 1.16. ([#78484](https://github.com/kubernetes/kubernetes/pull/78484), [@egernst](https://github.com/egernst))\n\n  Courtesy of SIG API Machinery, and SIG Node",
-    "author": "egernst",
-    "author_url": "https://github.com/egernst",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78484",
-    "pr_number": 78484,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "node"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "78495": {
-    "commit": "ca6113fb8bc0752aaaa2956b7d900059acf138f5",
-    "text": "Fix kubelet errors in AArch64 with huge page sizes smaller than 1MiB",
-    "markdown": "Fix kubelet errors in AArch64 with huge page sizes smaller than 1MiB ([#78495](https://github.com/kubernetes/kubernetes/pull/78495), [@odinuge](https://github.com/odinuge))",
-    "author": "odinuge",
-    "author_url": "https://github.com/odinuge",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78495",
-    "pr_number": 78495,
-    "areas": ["apiserver", "dependency", "kubectl", "kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli", "node", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "78502": {
-    "commit": "f44d8f55ca1308ef576a15d264b23d4a00f77e8b",
-    "text": "Service account tokens now include the JWT Key ID field in their header.",
-    "markdown": "Service account tokens now include the JWT Key ID field in their header. ([#78502](https://github.com/kubernetes/kubernetes/pull/78502), [@ahmedtd](https://github.com/ahmedtd))\n\n  Courtesy of SIG Auth",
-    "author": "ahmedtd",
-    "author_url": "https://github.com/ahmedtd",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78502",
-    "pr_number": 78502,
-    "kinds": ["feature"],
-    "sigs": ["auth"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "78513": {
-    "commit": "6efc5a8c9aa9d0c6509b59cb5e2761a56cf4f860",
-    "text": "Add Bind extension point of the scheduling framework",
-    "markdown": "Add Bind extension point of the scheduling framework ([#78513](https://github.com/kubernetes/kubernetes/pull/78513), [@chenchun](https://github.com/chenchun))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "chenchun",
-    "author_url": "https://github.com/chenchun",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78513",
-    "pr_number": 78513,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "78540": {
-    "commit": "a7cde2e70fc2c82ada19367b65a27e9a67173038",
-    "text": "KMS Providers will install a healthz check for the status of kms-pluign in kube-apiservers' encryption config.",
-    "markdown": "KMS Providers will install a healthz check for the status of kms-pluign in kube-apiservers' encryption config. ([#78540](https://github.com/kubernetes/kubernetes/pull/78540), [@immutableT](https://github.com/immutableT))\n\n  Courtesy of SIG API Machinery, SIG Auth, SIG CLI, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Node, and SIG Testing",
-    "author": "immutableT",
-    "author_url": "https://github.com/immutableT",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78540",
-    "pr_number": 78540,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": [
-      "api-machinery",
-      "auth",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "node",
-      "testing"
-    ],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "78687": {
-    "commit": "0e499be526bc91ad2d2e187428da12daa03a1eae",
-    "text": "fix pod list return value of framework.WaitForPodsWithLabelRunningReady",
-    "markdown": "fix pod list return value of framework.WaitForPodsWithLabelRunningReady ([#78687](https://github.com/kubernetes/kubernetes/pull/78687), [@pohly](https://github.com/pohly))",
-    "author": "pohly",
-    "author_url": "https://github.com/pohly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78687",
-    "pr_number": 78687,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["bug"],
-    "sigs": ["testing"],
-    "release_version": "1.16.0"
-  },
-  "78732": {
-    "commit": "74b5603f8326c7fcaca29e6a0d6827ca29863433",
-    "text": "Added a metric 'apiserver_watch_events_total' that can be used to understand the number of watch events in the system.",
-    "markdown": "Added a metric 'apiserver_watch_events_total' that can be used to understand the number of watch events in the system. ([#78732](https://github.com/kubernetes/kubernetes/pull/78732), [@mborsz](https://github.com/mborsz))\n\n  Courtesy of SIG API Machinery",
-    "author": "mborsz",
-    "author_url": "https://github.com/mborsz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78732",
-    "pr_number": 78732,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "78742": {
-    "commit": "91b26341f47088896ad373f4df8c7bdf5b249bcc",
-    "text": "The garbage collector and generic object quota controller have been updated to use the metadata client which improves memory\nand CPU usage of the Kube controller manager.",
-    "markdown": "The garbage collector and generic object quota controller have been updated to use the metadata client which improves memory\n  and CPU usage of the Kube controller manager. ([#78742](https://github.com/kubernetes/kubernetes/pull/78742), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78742",
-    "pr_number": 78742,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "apps", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "78744": {
-    "commit": "ca6cc69bd4e1e5b72b94863729a5a10936684867",
-    "text": "The namespace controller has been updated to use the metadata client which improves memory\nand CPU usage of the Kube controller manager.",
-    "markdown": "The namespace controller has been updated to use the metadata client which improves memory\n  and CPU usage of the Kube controller manager. ([#78744](https://github.com/kubernetes/kubernetes/pull/78744), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78744",
-    "pr_number": 78744,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "apps", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "78775": {
-    "commit": "a7c81c663d4a59d0e4d9be96c394b7dad1a6cf76",
-    "text": "kube-proxy --cleanup will return the correct exit code if the cleanup was successful",
-    "markdown": "kube-proxy --cleanup will return the correct exit code if the cleanup was successful ([#78775](https://github.com/kubernetes/kubernetes/pull/78775), [@johscheuer](https://github.com/johscheuer))",
-    "author": "johscheuer",
-    "author_url": "https://github.com/johscheuer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78775",
-    "pr_number": 78775,
-    "kinds": ["cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.16.0"
-  },
-  "78793": {
-    "commit": "c63000ef81eac07508892e0aac9208b498ac9896",
-    "text": "When specifying `--(kube|system)-reserved-cgroup`, with `--cgroup-driver=systemd`, it is now possible to use the fully qualified cgroupfs name (i.e. `/test-cgroup.slice`).",
-    "markdown": "When specifying `--(kube|system)-reserved-cgroup`, with `--cgroup-driver=systemd`, it is now possible to use the fully qualified cgroupfs name (i.e. `/test-cgroup.slice`). ([#78793](https://github.com/kubernetes/kubernetes/pull/78793), [@mattjmcnaughton](https://github.com/mattjmcnaughton))",
-    "author": "mattjmcnaughton",
-    "author_url": "https://github.com/mattjmcnaughton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78793",
-    "pr_number": 78793,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.16.0"
-  },
-  "78829": {
-    "commit": "f9afe46d23016a010a2c34b65fa312cde90153ad",
-    "text": "Verify that CRD default values in OpenAPI specs are pruned, with the exceptions of values under `metadata`.",
-    "markdown": "Verify that CRD default values in OpenAPI specs are pruned, with the exceptions of values under `metadata`. ([#78829](https://github.com/kubernetes/kubernetes/pull/78829), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78829",
-    "pr_number": 78829,
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.0"
-  },
-  "78868": {
-    "commit": "f4a62ad660944e13f51b95cedfe15742580472fb",
-    "text": "Mounts /home/kubernetes/bin/nvidia/vulkan/icd.d on the host to /etc/vulkan/icd.d inside containers requesting GPU.",
-    "markdown": "Mounts /home/kubernetes/bin/nvidia/vulkan/icd.d on the host to /etc/vulkan/icd.d inside containers requesting GPU. ([#78868](https://github.com/kubernetes/kubernetes/pull/78868), [@chardch](https://github.com/chardch))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "chardch",
-    "author_url": "https://github.com/chardch",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78868",
-    "pr_number": 78868,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "78908": {
-    "commit": "6f73ab22197ad0753a01f76f6f86caa03a51f548",
-    "text": "kubelet now accepts a --cni-cache-dir option, which defaults to /var/lib/cni/cache, where CNI stores cache files.",
-    "markdown": "kubelet now accepts a --cni-cache-dir option, which defaults to /var/lib/cni/cache, where CNI stores cache files. ([#78908](https://github.com/kubernetes/kubernetes/pull/78908), [@dcbw](https://github.com/dcbw))",
-    "author": "dcbw",
-    "author_url": "https://github.com/dcbw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78908",
-    "pr_number": 78908,
-    "areas": ["dependency", "kubeadm", "kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "network", "node", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "78941": {
-    "commit": "fde94e931f90d3a0d01808caf0b2e94e9ea8c629",
-    "text": "Failed iscsi logout is now re-tried periodically.",
-    "markdown": "Failed iscsi logout is now re-tried periodically. ([#78941](https://github.com/kubernetes/kubernetes/pull/78941), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78941",
-    "pr_number": 78941,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "78958": {
-    "commit": "bd35a48815a46191e47f2e139f0fb90e4cc8eb15",
-    "text": "update to use go 1.12.6",
-    "markdown": "update to use go 1.12.6 ([#78958](https://github.com/kubernetes/kubernetes/pull/78958), [@tao12345666333](https://github.com/tao12345666333))\n\n  Courtesy of SIG Testing",
-    "author": "tao12345666333",
-    "author_url": "https://github.com/tao12345666333",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78958",
-    "pr_number": 78958,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "78974": {
-    "commit": "1782653fe87245fdbd49f96dbeee3bb4db13fa2e",
-    "text": "fix a bug that pods not be deleted from unmatched nodes by daemon controller",
-    "markdown": "fix a bug that pods not be deleted from unmatched nodes by daemon controller ([#78974](https://github.com/kubernetes/kubernetes/pull/78974), [@DaiHao](https://github.com/DaiHao))",
-    "author": "DaiHao",
-    "author_url": "https://github.com/DaiHao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78974",
-    "pr_number": 78974,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.16.0"
-  },
-  "78991": {
-    "commit": "1a15ce14473fb60bb817d1f29147ae7b70a0987e",
-    "text": "Reflector watchHandler Warning log 'The resourceVersion for the provided watch is too old.' is now logged as Info.",
-    "markdown": "Reflector watchHandler Warning log 'The resourceVersion for the provided watch is too old.' is now logged as Info. ([#78991](https://github.com/kubernetes/kubernetes/pull/78991), [@sallyom](https://github.com/sallyom))",
-    "author": "sallyom",
-    "author_url": "https://github.com/sallyom",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78991",
-    "pr_number": 78991,
-    "areas": ["apiserver", "cloudprovider", "dependency"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "78999": {
-    "commit": "101f9ff7031f02aec7c2b6fdb9001a7fc0b630a7",
-    "text": "Fix a string comparison bug in IPVS graceful termination where UDP real servers are not deleted.",
-    "markdown": "Fix a string comparison bug in IPVS graceful termination where UDP real servers are not deleted. ([#78999](https://github.com/kubernetes/kubernetes/pull/78999), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78999",
-    "pr_number": 78999,
-    "areas": ["ipvs"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.16.0"
-  },
-  "79000": {
-    "commit": "af9d54db76624c04e014c0d6d2146f83781d4617",
-    "text": "Fixes a bug where `kubectl set config` hangs and uses 100% CPU on some invalid property names",
-    "markdown": "Fixes a bug where `kubectl set config` hangs and uses 100% CPU on some invalid property names ([#79000](https://github.com/kubernetes/kubernetes/pull/79000), [@pswica](https://github.com/pswica))",
-    "author": "pswica",
-    "author_url": "https://github.com/pswica",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79000",
-    "pr_number": 79000,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.16.0"
-  },
-  "79002": {
-    "commit": "6602f6159660b8fa17c743246f37acfeb3baee5e",
-    "text": "Enable cadvisor ProcessMetrics collecting.",
-    "markdown": "Enable cadvisor ProcessMetrics collecting. ([#79002](https://github.com/kubernetes/kubernetes/pull/79002), [@jiayingz](https://github.com/jiayingz))\n\n  Courtesy of SIG Node",
-    "author": "jiayingz",
-    "author_url": "https://github.com/jiayingz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79002",
-    "pr_number": 79002,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "79014": {
-    "commit": "042b157d45ba671f77ffa14821b8a3b09b94127c",
-    "text": "updated fluentd to 1.5.1, elasticsearchs \u0026 kibana to 7.1.1",
-    "markdown": "updated fluentd to 1.5.1, elasticsearchs \u0026 kibana to 7.1.1 ([#79014](https://github.com/kubernetes/kubernetes/pull/79014), [@monotek](https://github.com/monotek))",
-    "author": "monotek",
-    "author_url": "https://github.com/monotek",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79014",
-    "pr_number": 79014,
-    "kinds": ["flake"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "79020": {
-    "commit": "9120d427dab4a69ddcf22597f81b574fb1af96bc",
-    "text": "fix: make azure disk URI as case insensitive",
-    "markdown": "fix: make azure disk URI as case insensitive ([#79020](https://github.com/kubernetes/kubernetes/pull/79020), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79020",
-    "pr_number": 79020,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.0"
-  },
-  "79033": {
-    "commit": "e6d427339593383df34ade066f7d571193686aae",
-    "text": "kubeadm: use the --pod-network-cidr flag to init or use the podSubnet field in the kubeadm config to pass a comma separated list of pod CIDRs.",
-    "markdown": "kubeadm: use the --pod-network-cidr flag to init or use the podSubnet field in the kubeadm config to pass a comma separated list of pod CIDRs. ([#79033](https://github.com/kubernetes/kubernetes/pull/79033), [@Arvinderpal](https://github.com/Arvinderpal))\n\n  Courtesy of SIG Cluster Lifecycle, SIG Network, and SIG Testing",
-    "author": "Arvinderpal",
-    "author_url": "https://github.com/Arvinderpal",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79033",
-    "pr_number": 79033,
-    "areas": ["kubeadm", "test"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "network", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "79073": {
-    "commit": "caf6b67ce2a2d2cd18b9634c51198275ffe638ad",
-    "text": "Remove pids cgroup controller requirement when related feature gates are disabled",
-    "markdown": "Remove pids cgroup controller requirement when related feature gates are disabled ([#79073](https://github.com/kubernetes/kubernetes/pull/79073), [@rafatio](https://github.com/rafatio))",
-    "author": "rafatio",
-    "author_url": "https://github.com/rafatio",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79073",
-    "pr_number": 79073,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.16.0"
-  },
-  "79094": {
-    "commit": "efa043151c458bc36c28931f8a69693b6d454bb7",
-    "text": "fix kubelet fail to delete orphaned pod directory when the kubelet's pods directory (default is \"/var/lib/kubelet/pods\") symbolically links to another disk device's directory",
-    "markdown": "fix kubelet fail to delete orphaned pod directory when the kubelet's pods directory (default is \"/var/lib/kubelet/pods\") symbolically links to another disk device's directory ([#79094](https://github.com/kubernetes/kubernetes/pull/79094), [@gaorong](https://github.com/gaorong))",
-    "author": "gaorong",
-    "author_url": "https://github.com/gaorong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79094",
-    "pr_number": 79094,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.16.0"
-  },
-  "79106": {
-    "commit": "c8b45cd25c18e65798dde49fc7011495ea6021d5",
-    "text": "The behavior of the default handler for 404 requests fro the GCE Ingress load balancer is slightly modified in the sense that it now exports metrics using prometheus. The metrics exported include: \n- http_404_request_total  (the number of 404 requests handled)\n- http_404_request_duration_ms (the amount of time the server took to respond in ms)\n\nAlso includes percentile groupings. The directory for the default 404 handler includes instructions on how to enable prometheus for monitoring and setting alerts.",
-    "markdown": "The behavior of the default handler for 404 requests fro the GCE Ingress load balancer is slightly modified in the sense that it now exports metrics using prometheus. The metrics exported include: \n  - http_404_request_total  (the number of 404 requests handled)\n  - http_404_request_duration_ms (the amount of time the server took to respond in ms)\n  \n  Also includes percentile groupings. The directory for the default 404 handler includes instructions on how to enable prometheus for monitoring and setting alerts. ([#79106](https://github.com/kubernetes/kubernetes/pull/79106), [@vbannai](https://github.com/vbannai))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "vbannai",
-    "author_url": "https://github.com/vbannai",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79106",
-    "pr_number": 79106,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "79109": {
-    "commit": "a3898dc41db8f9d8d25ba95b3c92eb4e5d9e2d64",
-    "text": "Add Score extension point to the scheduling framework.",
-    "markdown": "Add Score extension point to the scheduling framework. ([#79109](https://github.com/kubernetes/kubernetes/pull/79109), [@ahg-g](https://github.com/ahg-g))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "ahg-g",
-    "author_url": "https://github.com/ahg-g",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79109",
-    "pr_number": 79109,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "79114": {
-    "commit": "5cd9faf74a401df6845e34c857accfc5b2aa74c1",
-    "text": "The CRD handler now properly re-creates stale CR storage to reflect CRD update.",
-    "markdown": "The CRD handler now properly re-creates stale CR storage to reflect CRD update. ([#79114](https://github.com/kubernetes/kubernetes/pull/79114), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79114",
-    "pr_number": 79114,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.0"
-  },
-  "79165": {
-    "commit": "4683054ff6b476b8ebaac25460e76c65f528e1a0",
-    "text": "kubeadm: fix a potential panic if kubeadm discovers an invalid, existing kubeconfig file",
-    "markdown": "kubeadm: fix a potential panic if kubeadm discovers an invalid, existing kubeconfig file ([#79165](https://github.com/kubernetes/kubernetes/pull/79165), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79165",
-    "pr_number": 79165,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "79247": {
-    "commit": "8cf05f514c139b664f13687b8ac3c24fb99311ff",
-    "text": "Introduce support for applying pod overhead to pod cgroups, if the PodOverhead feature is enabled.",
-    "markdown": "Introduce support for applying pod overhead to pod cgroups, if the PodOverhead feature is enabled. ([#79247](https://github.com/kubernetes/kubernetes/pull/79247), [@egernst](https://github.com/egernst))\n\n  Courtesy of SIG Node",
-    "author": "egernst",
-    "author_url": "https://github.com/egernst",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79247",
-    "pr_number": 79247,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "79262": {
-    "commit": "8ce45b642ef6cdc45ca73301effd7837b663e7f3",
-    "text": "The GA PodPriority feature gate is now on by default and cannot be disabled. The feature gate will be removed in v1.18.",
-    "markdown": "The GA PodPriority feature gate is now on by default and cannot be disabled. The feature gate will be removed in v1.18. ([#79262](https://github.com/kubernetes/kubernetes/pull/79262), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79262",
-    "pr_number": 79262,
-    "areas": ["apiserver", "kubelet", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "apps", "node", "scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "79263": {
-    "commit": "c2286aa9bdf32af3661c272adb7707ff3439f2ff",
-    "text": "Remove DirectCodecFactory(replace with serializer.WithoutConversionCodecFactory), DirectEncoder(replace with runtime.WithVersionEncoder) and DirectDecoder(replace with runtime.WithoutVersionDecoder). action required",
-    "markdown": "Remove DirectCodecFactory(replace with serializer.WithoutConversionCodecFactory), DirectEncoder(replace with runtime.WithVersionEncoder) and DirectDecoder(replace with runtime.WithoutVersionDecoder). action required ([#79263](https://github.com/kubernetes/kubernetes/pull/79263), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG API Machinery",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79263",
-    "pr_number": 79263,
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "action_required": true,
-    "release_version": "1.16.0"
-  },
-  "79270": {
-    "commit": "4c9fab5c4066e129fbd9b1ca505ef8ea440a2e50",
-    "text": "kubeadm: provide \"--control-plane-endpoint\" flag for `controlPlaneEndpoint`",
-    "markdown": "kubeadm: provide \"--control-plane-endpoint\" flag for `controlPlaneEndpoint` ([#79270](https://github.com/kubernetes/kubernetes/pull/79270), [@SataQiu](https://github.com/SataQiu))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79270",
-    "pr_number": 79270,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "79284": {
-    "commit": "e79dcc2174a996ddf85efecddb4f119e61b4d741",
-    "text": "When using the conformance test image, a new environment variable E2E_USE_GO_RUNNER will cause the tests to be run with the new Golang-based test runner rather than the current bash wrapper.",
-    "markdown": "When using the conformance test image, a new environment variable E2E_USE_GO_RUNNER will cause the tests to be run with the new Golang-based test runner rather than the current bash wrapper. ([#79284](https://github.com/kubernetes/kubernetes/pull/79284), [@johnSchnake](https://github.com/johnSchnake))\n\n  Courtesy of SIG Cluster Lifecycle, SIG Release, and SIG Testing",
-    "author": "johnSchnake",
-    "author_url": "https://github.com/johnSchnake",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79284",
-    "pr_number": 79284,
-    "areas": ["conformance"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "release", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "79305": {
-    "commit": "ed9f340addb4679565ccca27f438a6324e225d2a",
-    "text": "- Node labels `beta.kubernetes.io/metadata-proxy-ready`, `beta.kubernetes.io/metadata-proxy-ready` and `beta.kubernetes.io/kube-proxy-ds-ready` are no longer added on new nodes.\n- ip-mask-agent addon starts to use the label `node.kubernetes.io/masq-agent-ds-ready` instead of `beta.kubernetes.io/masq-agent-ds-ready` as its node selector.\n- kube-proxy addon starts to use the label `node.kubernetes.io/kube-proxy-ds-ready` instead of `beta.kubernetes.io/kube-proxy-ds-ready` as its node selector.\n- metadata-proxy addon starts to use the label `cloud.google.com/metadata-proxy-ready` instead of `beta.kubernetes.io/metadata-proxy-ready` as its node selector.\n- Kubelet removes the ability to set `kubernetes.io` or `k8s.io` labels via --node-labels other than the [specifically allowed labels/prefixes](https://github.com/kubernetes/enhancements/blob/master/keps/sig-auth/0000-20170814-bounding-self-labeling-kubelets.md\u0026#35;proposal).",
-    "markdown": "- Node labels `beta.kubernetes.io/metadata-proxy-ready`, `beta.kubernetes.io/metadata-proxy-ready` and `beta.kubernetes.io/kube-proxy-ds-ready` are no longer added on new nodes.\n  - ip-mask-agent addon starts to use the label `node.kubernetes.io/masq-agent-ds-ready` instead of `beta.kubernetes.io/masq-agent-ds-ready` as its node selector.\n  - kube-proxy addon starts to use the label `node.kubernetes.io/kube-proxy-ds-ready` instead of `beta.kubernetes.io/kube-proxy-ds-ready` as its node selector.\n  - metadata-proxy addon starts to use the label `cloud.google.com/metadata-proxy-ready` instead of `beta.kubernetes.io/metadata-proxy-ready` as its node selector.\n  - Kubelet removes the ability to set `kubernetes.io` or `k8s.io` labels via --node-labels other than the [specifically allowed labels/prefixes](https://github.com/kubernetes/enhancements/blob/master/keps/sig-auth/0000-20170814-bounding-self-labeling-kubelets.md\u0026#35;proposal). ([#79305](https://github.com/kubernetes/kubernetes/pull/79305), [@paivagustavo](https://github.com/paivagustavo))\n\n  Courtesy of SIG Auth, SIG Cluster Lifecycle, and SIG Node",
-    "author": "paivagustavo",
-    "author_url": "https://github.com/paivagustavo",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79305",
-    "pr_number": 79305,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["auth", "cluster-lifecycle", "node"],
-    "action_required": true,
-    "release_version": "1.16.0"
-  },
-  "79307": {
-    "commit": "46469ee9d51d8e5760674bfca6ba2e374a9ab511",
-    "text": "Remove HugePages, VolumeScheduling, CustomPodDNS and PodReadinessGates feature flags",
-    "markdown": "Remove HugePages, VolumeScheduling, CustomPodDNS and PodReadinessGates feature flags ([#79307](https://github.com/kubernetes/kubernetes/pull/79307), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79307",
-    "pr_number": 79307,
-    "kinds": ["cleanup"],
-    "sigs": ["release"],
-    "release_version": "1.16.0"
-  },
-  "79309": {
-    "commit": "ad095324bfa99018e35fbac6d80c418d55c81064",
-    "text": "Remove CSIPersistentVolume feature gates",
-    "markdown": "Remove CSIPersistentVolume feature gates ([#79309](https://github.com/kubernetes/kubernetes/pull/79309), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79309",
-    "pr_number": 79309,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "apps", "auth", "node", "storage"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "79310": {
-    "commit": "2109c1a7a300e0ff41c7c0acd2dea00ba172c4b8",
-    "text": "Remove KubeletPluginsWatcher feature gates",
-    "markdown": "Remove KubeletPluginsWatcher feature gates ([#79310](https://github.com/kubernetes/kubernetes/pull/79310), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79310",
-    "pr_number": 79310,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["auth", "node", "storage"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "79313": {
-    "commit": "fc5ef37db177158b7f0afce5bb59908803193bf7",
-    "text": "Add Bind extension point to the scheduling framework.",
-    "markdown": "Add Bind extension point to the scheduling framework. ([#79313](https://github.com/kubernetes/kubernetes/pull/79313), [@chenchun](https://github.com/chenchun))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "chenchun",
-    "author_url": "https://github.com/chenchun",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79313",
-    "pr_number": 79313,
-    "areas": ["test"],
-    "kinds": ["feature", "flake"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "79326": {
-    "commit": "80a3177f17d9dc55f62cbf170c864f18b58c4fec",
-    "text": "Fix remove the etcd member from the cluster during a kubeadm reset.",
-    "markdown": "Fix remove the etcd member from the cluster during a kubeadm reset. ([#79326](https://github.com/kubernetes/kubernetes/pull/79326), [@bradbeam](https://github.com/bradbeam))",
-    "author": "bradbeam",
-    "author_url": "https://github.com/bradbeam",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79326",
-    "pr_number": 79326,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "79345": {
-    "commit": "47459465254b840b16220a163540631e416b22c2",
-    "text": "Fixes output of `kubectl get --watch-only` when watching a single resource",
-    "markdown": "Fixes output of `kubectl get --watch-only` when watching a single resource ([#79345](https://github.com/kubernetes/kubernetes/pull/79345), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79345",
-    "pr_number": 79345,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.16.0"
-  },
-  "79346": {
-    "commit": "5267a3dc730ff606cb5b1d1a4ee05ef04ae7ed3b",
-    "text": "Correct the maximum allowed insecure bind port for the kube-scheduler and kube-apiserver to 65535.",
-    "markdown": "Correct the maximum allowed insecure bind port for the kube-scheduler and kube-apiserver to 65535. ([#79346](https://github.com/kubernetes/kubernetes/pull/79346), [@ncdc](https://github.com/ncdc))",
-    "author": "ncdc",
-    "author_url": "https://github.com/ncdc",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79346",
-    "pr_number": 79346,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "79349": {
-    "commit": "c1fc717ac666dabbe7b723b0388fb04e5aa7e0ca",
-    "text": "Fix a bug in server printer that could cause kube-apiserver to panic.",
-    "markdown": "Fix a bug in server printer that could cause kube-apiserver to panic. ([#79349](https://github.com/kubernetes/kubernetes/pull/79349), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79349",
-    "pr_number": 79349,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.16.0"
-  },
-  "79375": {
-    "commit": "46667d0bbbcee0d8435feed3fe2fcb8c8977f9ab",
-    "text": "RateLimiter add a context-aware method, fix client-go request goruntine backlog in async timeout scene.",
-    "markdown": "RateLimiter add a context-aware method, fix client-go request goruntine backlog in async timeout scene. ([#79375](https://github.com/kubernetes/kubernetes/pull/79375), [@answer1991](https://github.com/answer1991))\n\n  Courtesy of SIG API Machinery",
-    "author": "answer1991",
-    "author_url": "https://github.com/answer1991",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79375",
-    "pr_number": 79375,
-    "kinds": ["bug", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "79386": {
-    "commit": "550fb1bfc3c773febbb0811262fd7a8f7847bb20",
-    "text": "+ to run:\nMaster: convert service CIDR to list  `--service-cluster-ip-range=\u003cCIDR\u003e,\u003cCIDR\u003e` and make sure `IPv6DualStack` feature flag is turned on. The flag is validated and used as the following:\n\n1. `--service-cluster-ip-range[0]` is consider primary service range, and will be used for any service with `Service.Spec.IPFamily = nil` or any service in the at the time of turning on the feature flag.\n2. A cluster can be dualstack (i.e. Pods and nodes carry dualstack IPs) but does not need to support ingress on dualstack. In this case the cluster can perform egress using `PodIPs` (according to family and binding selection in user code) but will ingress will only be performed against the pod primary IP. This can be configured by supplying single entry to `--service-cluster-ip-range` flag.\n3. Maximum of two entries is allowed in `--service-cluster-ip-range` and they are validated to be dual stacked `i.e. --service-cluster-ip-range=\u003cv4\u003e,\u003cv6\u003e or --service-cluster-ip-range=\u003cv6\u003e,\u003cv4\u003e`  \n4. Max 20 bit for range (min network bits `\u003cv6\u003e/108` or \u003cv4\u003e/12)\n\nkube-controller-manager: convert service CIDR to list `--service-cluster-ip-range=\u003cCIDR\u003e,\u003cCIDR\u003e` and make sure `IPv6DualStack` feature flag is turned on. The flag is validated as above.\n\n+ to use:\nA new service spec field `Service.Spec.IPFamily` has been added. The default of this field is family of (first service cidr in --service-cluster-ip-range flag). The value is defaulted as described above  once the feature gate is turned on. Here are the possible values for this field:\n2. IPv4: api-server will assign an IP from a `service-cluster-ip-range` that is `ipv4` (either the primary or the secondary, according to how they were configured).\n2. IPv6: api-server will assign an IP from a `service-cluster-ip-range` that is `ipv6` (either the primary or the secondary, according to how they were configured).\n\nNotes (v1.16):\n1. IPVS is the only proxy supported (as of v1.16 ) by Dualstack.\n2. Dualstack is mutually exclusive with `EndpointSlice` feature. They can not be turned on together.  `metaproxy` is yet to implement EndpointSlice handling.",
-    "markdown": "+ to run:\n  Master: convert service CIDR to list  `--service-cluster-ip-range=\u003cCIDR\u003e,\u003cCIDR\u003e` and make sure `IPv6DualStack` feature flag is turned on. The flag is validated and used as the following:\n  \n  1. `--service-cluster-ip-range[0]` is consider primary service range, and will be used for any service with `Service.Spec.IPFamily = nil` or any service in the at the time of turning on the feature flag.\n  2. A cluster can be dualstack (i.e. Pods and nodes carry dualstack IPs) but does not need to support ingress on dualstack. In this case the cluster can perform egress using `PodIPs` (according to family and binding selection in user code) but will ingress will only be performed against the pod primary IP. This can be configured by supplying single entry to `--service-cluster-ip-range` flag.\n  3. Maximum of two entries is allowed in `--service-cluster-ip-range` and they are validated to be dual stacked `i.e. --service-cluster-ip-range=\u003cv4\u003e,\u003cv6\u003e or --service-cluster-ip-range=\u003cv6\u003e,\u003cv4\u003e`  \n  4. Max 20 bit for range (min network bits `\u003cv6\u003e/108` or \u003cv4\u003e/12)\n  \n  kube-controller-manager: convert service CIDR to list `--service-cluster-ip-range=\u003cCIDR\u003e,\u003cCIDR\u003e` and make sure `IPv6DualStack` feature flag is turned on. The flag is validated as above.\n  \n  + to use:\n  A new service spec field `Service.Spec.IPFamily` has been added. The default of this field is family of (first service cidr in --service-cluster-ip-range flag). The value is defaulted as described above  once the feature gate is turned on. Here are the possible values for this field:\n  2. IPv4: api-server will assign an IP from a `service-cluster-ip-range` that is `ipv4` (either the primary or the secondary, according to how they were configured).\n  2. IPv6: api-server will assign an IP from a `service-cluster-ip-range` that is `ipv6` (either the primary or the secondary, according to how they were configured).\n  \n  Notes (v1.16):\n  1. IPVS is the only proxy supported (as of v1.16 ) by Dualstack.\n  2. Dualstack is mutually exclusive with `EndpointSlice` feature. They can not be turned on together.  `metaproxy` is yet to implement EndpointSlice handling. ([#79386](https://github.com/kubernetes/kubernetes/pull/79386), [@khenidak](https://github.com/khenidak))",
-    "author": "khenidak",
-    "author_url": "https://github.com/khenidak",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79386",
-    "pr_number": 79386,
-    "areas": [
-      "apiserver",
-      "cloudprovider",
-      "code-generation",
-      "dependency",
-      "kubectl",
-      "kubelet",
-      "release-eng",
-      "test"
-    ],
-    "kinds": ["api-change"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "network",
-      "node",
-      "release",
-      "scheduling",
-      "testing"
-    ],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "79391": {
-    "commit": "c8cee54240a0f36264b4de62423617dbc30b57aa",
-    "text": "Kubelet should now more reliably report the same primary node IP even if the set of node IPs reported by the CloudProvider changes.",
-    "markdown": "Kubelet should now more reliably report the same primary node IP even if the set of node IPs reported by the CloudProvider changes. ([#79391](https://github.com/kubernetes/kubernetes/pull/79391), [@danwinship](https://github.com/danwinship))",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79391",
-    "pr_number": 79391,
-    "areas": ["kubelet"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["node"],
-    "release_version": "1.16.0"
-  },
-  "79444": {
-    "commit": "1dac5fd14a54ac4972339dbe55f9f03688fd7542",
-    "text": "Reduce kube-proxy cpu usage in IPVS mode when a large number of nodePort services exist.",
-    "markdown": "Reduce kube-proxy cpu usage in IPVS mode when a large number of nodePort services exist. ([#79444](https://github.com/kubernetes/kubernetes/pull/79444), [@cezarsa](https://github.com/cezarsa))",
-    "author": "cezarsa",
-    "author_url": "https://github.com/cezarsa",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79444",
-    "pr_number": 79444,
-    "areas": ["ipvs"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.16.0"
-  },
-  "79451": {
-    "commit": "b51f62186a75650f0021da9e3715c90609984cfc",
-    "text": "Fix a bug where kubelet would not retry pod sandbox creation when the restart policy of the pod is Never",
-    "markdown": "Fix a bug where kubelet would not retry pod sandbox creation when the restart policy of the pod is Never ([#79451](https://github.com/kubernetes/kubernetes/pull/79451), [@yujuhong](https://github.com/yujuhong))",
-    "author": "yujuhong",
-    "author_url": "https://github.com/yujuhong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79451",
-    "pr_number": 79451,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.16.0"
-  },
-  "79489": {
-    "commit": "2b21e478b1414846f06fcc59cc99940c36da8187",
-    "text": "Add v1.Container.SecurityContext.WindowsOptions.RunAsUserName to the pod spec",
-    "markdown": "Add v1.Container.SecurityContext.WindowsOptions.RunAsUserName to the pod spec ([#79489](https://github.com/kubernetes/kubernetes/pull/79489), [@bclau](https://github.com/bclau))",
-    "author": "bclau",
-    "author_url": "https://github.com/bclau",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79489",
-    "pr_number": 79489,
-    "areas": ["kubelet", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["apps", "node", "testing", "windows"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "79493": {
-    "commit": "dd8a000a77341a11b1616b3ead341184764652f3",
-    "text": "Fix duplicate error messages in cli commands",
-    "markdown": "Fix duplicate error messages in cli commands ([#79493](https://github.com/kubernetes/kubernetes/pull/79493), [@odinuge](https://github.com/odinuge))",
-    "author": "odinuge",
-    "author_url": "https://github.com/odinuge",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79493",
-    "pr_number": 79493,
-    "areas": ["apiserver", "kubeadm", "kubectl", "kubelet"],
-    "kinds": ["bug"],
-    "sigs": [
-      "api-machinery",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "network",
-      "node",
-      "release",
-      "scheduling"
-    ],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "79498": {
-    "commit": "7db459a11bac8c092b0f319e4c5d6adaf2fe1912",
-    "text": "kubeadm: fix the bug that \"--cri-socket\" flag does not work for `kubeadm reset`",
-    "markdown": "kubeadm: fix the bug that \"--cri-socket\" flag does not work for `kubeadm reset` ([#79498](https://github.com/kubernetes/kubernetes/pull/79498), [@SataQiu](https://github.com/SataQiu))",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79498",
-    "pr_number": 79498,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "79504": {
-    "commit": "665e76d97623447d13bb3b33b68591a985b49c9d",
-    "text": "The alpha `metadata.initializers` field, deprecated in 1.13, has been removed.",
-    "markdown": "The alpha `metadata.initializers` field, deprecated in 1.13, has been removed. ([#79504](https://github.com/kubernetes/kubernetes/pull/79504), [@yue9944882](https://github.com/yue9944882))",
-    "author": "yue9944882",
-    "author_url": "https://github.com/yue9944882",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79504",
-    "pr_number": 79504,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.0"
-  },
-  "79514": {
-    "commit": "5019bb57a3e146bf7c19dfe178cbc34b456388dc",
-    "text": "Default resourceGroup should be used when the value of annotation azure-load-balancer-resource-group is an empty string.",
-    "markdown": "Default resourceGroup should be used when the value of annotation azure-load-balancer-resource-group is an empty string. ([#79514](https://github.com/kubernetes/kubernetes/pull/79514), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79514",
-    "pr_number": 79514,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.0"
-  },
-  "79529": {
-    "commit": "a3be4b6817ea7a62482196fe78f07e4fe7b5fde2",
-    "text": "changes timeout value in csi plugin from 15s to 2min which fixes the timeout issue",
-    "markdown": "changes timeout value in csi plugin from 15s to 2min which fixes the timeout issue ([#79529](https://github.com/kubernetes/kubernetes/pull/79529), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79529",
-    "pr_number": 79529,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.16.0"
-  },
-  "79534": {
-    "commit": "a5e3d7469149c704e32a9fd858be2a25c4ca9a2c",
-    "text": "Fix possible fd leak and closing of dirs in doSafeMakeDir",
-    "markdown": "Fix possible fd leak and closing of dirs in doSafeMakeDir ([#79534](https://github.com/kubernetes/kubernetes/pull/79534), [@odinuge](https://github.com/odinuge))",
-    "author": "odinuge",
-    "author_url": "https://github.com/odinuge",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79534",
-    "pr_number": 79534,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.16.0"
-  },
-  "79539": {
-    "commit": "6c45b3c23017bb232dc1db90e8fac914c9f2a705",
-    "text": "Added E2E tests validating WindowsOptions.RunAsUserName.",
-    "markdown": "Added E2E tests validating WindowsOptions.RunAsUserName. ([#79539](https://github.com/kubernetes/kubernetes/pull/79539), [@bclau](https://github.com/bclau))\n\n  Courtesy of SIG Apps, SIG Node, SIG Testing, and SIG Windows",
-    "author": "bclau",
-    "author_url": "https://github.com/bclau",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79539",
-    "pr_number": 79539,
-    "areas": ["kubelet", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["apps", "node", "testing", "windows"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "79549": {
-    "commit": "3dd8add5d25d29358fb239ac23461643f1d8effc",
-    "text": "The `MutatingWebhookConfiguration` and `ValidatingWebhookConfiguration` APIs have been promoted to `admissionregistration.k8s.io/v1`:\n- `failurePolicy` default changed from `Ignore` to `Fail` for v1\n- `matchPolicy` default changed from `Exact` to `Equivalent` for v1\n- `timeout` default changed from `30s` to `10s` for v1\n- `sideEffects` default value is removed, and the field made required, and only `None` and `NoneOnDryRun` are permitted for v1\n- `admissionReviewVersions` default value is removed and the field made required for v1 (supported versions for AdmissionReview are `v1` and `v1beta1`)\n- The `name` field for specified webhooks must be unique for `MutatingWebhookConfiguration` and `ValidatingWebhookConfiguration` objects created via `admissionregistration.k8s.io/v1`\nThe `admissionregistration.k8s.io/v1beta1` versions of `MutatingWebhookConfiguration` and `ValidatingWebhookConfiguration` are deprecated and will no longer be served in v1.19.",
-    "markdown": "The `MutatingWebhookConfiguration` and `ValidatingWebhookConfiguration` APIs have been promoted to `admissionregistration.k8s.io/v1`:\n  - `failurePolicy` default changed from `Ignore` to `Fail` for v1\n  - `matchPolicy` default changed from `Exact` to `Equivalent` for v1\n  - `timeout` default changed from `30s` to `10s` for v1\n  - `sideEffects` default value is removed, and the field made required, and only `None` and `NoneOnDryRun` are permitted for v1\n  - `admissionReviewVersions` default value is removed and the field made required for v1 (supported versions for AdmissionReview are `v1` and `v1beta1`)\n  - The `name` field for specified webhooks must be unique for `MutatingWebhookConfiguration` and `ValidatingWebhookConfiguration` objects created via `admissionregistration.k8s.io/v1`\n  The `admissionregistration.k8s.io/v1beta1` versions of `MutatingWebhookConfiguration` and `ValidatingWebhookConfiguration` are deprecated and will no longer be served in v1.19. ([#79549](https://github.com/kubernetes/kubernetes/pull/79549), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG CLI, and SIG Testing",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79549",
-    "pr_number": 79549,
-    "areas": ["dependency", "kubectl", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "cli", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "79552": {
-    "commit": "60c2d44129ba238ceb548467b3931c9b76032965",
-    "text": "Add support for AWS EBS on windows",
-    "markdown": "Add support for AWS EBS on windows ([#79552](https://github.com/kubernetes/kubernetes/pull/79552), [@wongma7](https://github.com/wongma7))\n\n  Courtesy of SIG Storage, SIG Testing, and SIG Windows",
-    "author": "wongma7",
-    "author_url": "https://github.com/wongma7",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79552",
-    "pr_number": 79552,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["storage", "testing", "windows"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "79574": {
-    "commit": "1ff4525a29f1736fca19e10b49e42664b29010c2",
-    "text": "Update Azure SDK + go-autorest API versions",
-    "markdown": "Update Azure SDK + go-autorest API versions ([#79574](https://github.com/kubernetes/kubernetes/pull/79574), [@justaugustus](https://github.com/justaugustus))\n\n  Courtesy of SIG API Machinery, SIG CLI, SIG Cloud Provider, SIG Cluster Lifecycle, and SIG Node",
-    "author": "justaugustus",
-    "author_url": "https://github.com/justaugustus",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79574",
-    "pr_number": 79574,
-    "areas": [
-      "apiserver",
-      "cloudprovider",
-      "code-generation",
-      "dependency",
-      "kubectl",
-      "provider/azure"
-    ],
-    "kinds": ["cleanup", "feature"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "node"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "79583": {
-    "commit": "38e8b8a9f65e66b779a0985e17868b036c1d601b",
-    "text": "Update Azure API versions (containerregistry --\u003e 2018-09-01, network --\u003e 2018-08-01)",
-    "markdown": "Update Azure API versions (containerregistry --\u003e 2018-09-01, network --\u003e 2018-08-01) ([#79583](https://github.com/kubernetes/kubernetes/pull/79583), [@justaugustus](https://github.com/justaugustus))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "justaugustus",
-    "author_url": "https://github.com/justaugustus",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79583",
-    "pr_number": 79583,
-    "areas": ["cloudprovider"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "79604": {
-    "commit": "ef397e5b368d530e712165a4bb78ec482d739c2b",
-    "text": "The `CustomResourceDefinition` API type is promoted to `apiextensions.k8s.io/v1` with the following changes:\n- Use of the new `default` feature in validation schemas is limited to v1\n- `spec.scope` is no longer defaulted to `Namespaced` and must be explicitly specified\n- `spec.version` is removed; use `spec.versions` instead\n- `spec.validation` is removed; use `spec.versions[*].schema` instead\n- `spec.subresources` is removed; use `spec.versions[*].subresources` instead\n- `spec.additionalPrinterColumns` is removed; use `spec.versions[*].additionalPrinterColumns` instead\n- `spec.conversion.webhookClientConfig` is moved to `spec.conversion.webhook.clientConfig`\n- `spec.conversion.conversionReviewVersions` is moved to `spec.conversion.webhook.conversionReviewVersions`\n- `spec.versions[*].schema.openAPIV3Schema` is now required when creating v1 CustomResourceDefinitions\n- `spec.preserveUnknownFields: true` is disallowed when creating v1 CustomResourceDefinitions; it must be specified within schema definitions as `x-kubernetes-preserve-unknown-fields: true`\n- In `additionalPrinterColumns` items, the `JSONPath` field was renamed to `jsonPath` (fixes https://github.com/kubernetes/kubernetes/issues/66531)\nThe `apiextensions.k8s.io/v1beta1` version of `CustomResourceDefinition` is deprecated and will no longer be served in v1.19.",
-    "markdown": "The `CustomResourceDefinition` API type is promoted to `apiextensions.k8s.io/v1` with the following changes:\n  - Use of the new `default` feature in validation schemas is limited to v1\n  - `spec.scope` is no longer defaulted to `Namespaced` and must be explicitly specified\n  - `spec.version` is removed; use `spec.versions` instead\n  - `spec.validation` is removed; use `spec.versions[*].schema` instead\n  - `spec.subresources` is removed; use `spec.versions[*].subresources` instead\n  - `spec.additionalPrinterColumns` is removed; use `spec.versions[*].additionalPrinterColumns` instead\n  - `spec.conversion.webhookClientConfig` is moved to `spec.conversion.webhook.clientConfig`\n  - `spec.conversion.conversionReviewVersions` is moved to `spec.conversion.webhook.conversionReviewVersions`\n  - `spec.versions[*].schema.openAPIV3Schema` is now required when creating v1 CustomResourceDefinitions\n  - `spec.preserveUnknownFields: true` is disallowed when creating v1 CustomResourceDefinitions; it must be specified within schema definitions as `x-kubernetes-preserve-unknown-fields: true`\n  - In `additionalPrinterColumns` items, the `JSONPath` field was renamed to `jsonPath` (fixes https://github.com/kubernetes/kubernetes/issues/66531)\n  The `apiextensions.k8s.io/v1beta1` version of `CustomResourceDefinition` is deprecated and will no longer be served in v1.19. ([#79604](https://github.com/kubernetes/kubernetes/pull/79604), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery, and SIG Testing",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79604",
-    "pr_number": 79604,
-    "areas": ["apiserver", "custom-resources", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "79636": {
-    "commit": "8765ed365f9e206fd1296aaa216e3c9346671e40",
-    "text": "Fixes a bug in openapi published for custom resources using x-kubernetes-preserve-unknown-fields extensions, so that kubectl will allow sending unknown fields for that portion of the object.",
-    "markdown": "Fixes a bug in openapi published for custom resources using x-kubernetes-preserve-unknown-fields extensions, so that kubectl will allow sending unknown fields for that portion of the object. ([#79636](https://github.com/kubernetes/kubernetes/pull/79636), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79636",
-    "pr_number": 79636,
-    "areas": ["custom-resources", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "79641": {
-    "commit": "2ad2795136be1edd0d1920e7b3e8c25e1e66f6a4",
-    "text": "If scheduler extender filtered a not found node, current scheduling round for this pod will just be skipped.",
-    "markdown": "If scheduler extender filtered a not found node, current scheduling round for this pod will just be skipped. ([#79641](https://github.com/kubernetes/kubernetes/pull/79641), [@yqwang-ms](https://github.com/yqwang-ms))",
-    "author": "yqwang-ms",
-    "author_url": "https://github.com/yqwang-ms",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79641",
-    "pr_number": 79641,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.16.0"
-  },
-  "79677": {
-    "commit": "7340b6341a4e867cfc5761e8642c54c55ff6db3e",
-    "text": "kubeadm: implement support for concurrent add/remove of stacked etcd members",
-    "markdown": "kubeadm: implement support for concurrent add/remove of stacked etcd members ([#79677](https://github.com/kubernetes/kubernetes/pull/79677), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79677",
-    "pr_number": 79677,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "79722": {
-    "commit": "133f37814c0ad5d140f9fcb7c53e467cebeed65d",
-    "text": "etcd Docker image can be run as non-root",
-    "markdown": "etcd Docker image can be run as non-root ([#79722](https://github.com/kubernetes/kubernetes/pull/79722), [@randomvariable](https://github.com/randomvariable))",
-    "author": "randomvariable",
-    "author_url": "https://github.com/randomvariable",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79722",
-    "pr_number": 79722,
-    "areas": ["security"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "release"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "79724": {
-    "commit": "b130ff450232eaf9100548aa08295168871a4ee9",
-    "text": "add  `kubectl replace --raw` and `kubectl delete --raw` to have parity with create and get",
-    "markdown": "add  `kubectl replace --raw` and `kubectl delete --raw` to have parity with create and get ([#79724](https://github.com/kubernetes/kubernetes/pull/79724), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79724",
-    "pr_number": 79724,
-    "areas": ["dependency", "kubectl", "release-eng", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cli", "release", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "79786": {
-    "commit": "a46eed0ca3988a8d1fba78e2b8f882df1f7406f4",
-    "text": "Promote WatchBookmark feature to beta and enable it by default.\nWith WatchBookmark feature, clients are able to request watch events with BOOKMARK type. Clients should not assume bookmarks are returned at any specific interval, nor may they assume the server will send any BOOKMARK event during a session.",
-    "markdown": "Promote WatchBookmark feature to beta and enable it by default.\n  With WatchBookmark feature, clients are able to request watch events with BOOKMARK type. Clients should not assume bookmarks are returned at any specific interval, nor may they assume the server will send any BOOKMARK event during a session. ([#79786](https://github.com/kubernetes/kubernetes/pull/79786), [@wojtek-t](https://github.com/wojtek-t))\n\n  Courtesy of SIG API Machinery",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79786",
-    "pr_number": 79786,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "79804": {
-    "commit": "a56e86a405a414f83e5bb1d4e84e37d80af543b7",
-    "text": "Extender bind should respect IsInterested",
-    "markdown": "Extender bind should respect IsInterested ([#79804](https://github.com/kubernetes/kubernetes/pull/79804), [@yqwang-ms](https://github.com/yqwang-ms))",
-    "author": "yqwang-ms",
-    "author_url": "https://github.com/yqwang-ms",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79804",
-    "pr_number": 79804,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.16.0"
-  },
-  "79843": {
-    "commit": "6cd0392ced5a14409c0dd074bdb8b9c33a433f06",
-    "text": "The sample-apiserver gains support for OpenAPI v2 spec serving at `/openapi/v2`.\n\nThe `generate-internal-groups.sh` script in k8s.io/code-generator will generate OpenAPI definitions by default in `pkg/generated/openapi`. Additional API group dependencies can be added via `OPENAPI_EXTRA_PACKAGES=\u003cgroup\u003e/\u003cversion\u003e \u003cgroup2\u003e/\u003cversion2\u003e...`.",
-    "markdown": "The sample-apiserver gains support for OpenAPI v2 spec serving at `/openapi/v2`.\n  \n  The `generate-internal-groups.sh` script in k8s.io/code-generator will generate OpenAPI definitions by default in `pkg/generated/openapi`. Additional API group dependencies can be added via `OPENAPI_EXTRA_PACKAGES=\u003cgroup\u003e/\u003cversion\u003e \u003cgroup2\u003e/\u003cversion2\u003e...`. ([#79843](https://github.com/kubernetes/kubernetes/pull/79843), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79843",
-    "pr_number": 79843,
-    "areas": ["code-generation", "dependency", "release-eng", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "release", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "79851": {
-    "commit": "0e3b593ded745f0aac2261cd9f8938e0d6f5556c",
-    "text": "Fixed a bug in the CSI metrics that does not return not supported error when a CSI driver does not support metrics.",
-    "markdown": "Fixed a bug in the CSI metrics that does not return not supported error when a CSI driver does not support metrics. ([#79851](https://github.com/kubernetes/kubernetes/pull/79851), [@jparklab](https://github.com/jparklab))",
-    "author": "jparklab",
-    "author_url": "https://github.com/jparklab",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79851",
-    "pr_number": 79851,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.16.0"
-  },
-  "79895": {
-    "commit": "262e59b2c04bf71ec4e56fdc2f3a91a5878336f4",
-    "text": "Resolves an issue serving aggregated APIs backed by services that respond to requests to `/` with non-2xx HTTP responses",
-    "markdown": "Resolves an issue serving aggregated APIs backed by services that respond to requests to `/` with non-2xx HTTP responses ([#79895](https://github.com/kubernetes/kubernetes/pull/79895), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79895",
-    "pr_number": 79895,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.0"
-  },
-  "79920": {
-    "commit": "10a1d1f5bdeb1a41438814489e2e594ab2805363",
-    "text": "Bugfix: csi plugin supporting raw block that does not need attach mounted failed",
-    "markdown": "Bugfix: csi plugin supporting raw block that does not need attach mounted failed ([#79920](https://github.com/kubernetes/kubernetes/pull/79920), [@cwdsuzhou](https://github.com/cwdsuzhou))",
-    "author": "cwdsuzhou",
-    "author_url": "https://github.com/cwdsuzhou",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79920",
-    "pr_number": 79920,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.16.0"
-  },
-  "79966": {
-    "commit": "824dfe93899e1d6da4ee6965db52f83eb56e06ac",
-    "text": "update to use go 1.12.7",
-    "markdown": "update to use go 1.12.7 ([#79966](https://github.com/kubernetes/kubernetes/pull/79966), [@tao12345666333](https://github.com/tao12345666333))\n\n  Courtesy of SIG Testing",
-    "author": "tao12345666333",
-    "author_url": "https://github.com/tao12345666333",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79966",
-    "pr_number": 79966,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "79983": {
-    "commit": "a3750501b081ecd3ca33df496575066da7449140",
-    "text": "when PodInfoOnMount is enabled for a CSI driver, the new csi.storage.k8s.io/ephemeral parameter in the volume context allows a driver's NodePublishVolume implementation to determine on a case-by-case basis whether the volume is ephemeral or a normal persistent volume",
-    "markdown": "when PodInfoOnMount is enabled for a CSI driver, the new csi.storage.k8s.io/ephemeral parameter in the volume context allows a driver's NodePublishVolume implementation to determine on a case-by-case basis whether the volume is ephemeral or a normal persistent volume ([#79983](https://github.com/kubernetes/kubernetes/pull/79983), [@pohly](https://github.com/pohly))\n\n  Courtesy of SIG Storage, and SIG Testing",
-    "author": "pohly",
-    "author_url": "https://github.com/pohly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79983",
-    "pr_number": 79983,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["storage", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "79992": {
-    "commit": "e0ff9634d5b5a480595930236c87dec1327dbf13",
-    "text": "CRDs under k8s.io and kubernetes.io must have the \"api-approved.kubernetes.io\" set to either `unapproved.*` or a link to the pull request approving the schema.  See https://github.com/kubernetes/enhancements/pull/1111 for more details.",
-    "markdown": "CRDs under k8s.io and kubernetes.io must have the \"api-approved.kubernetes.io\" set to either `unapproved.*` or a link to the pull request approving the schema.  See https://github.com/kubernetes/enhancements/pull/1111 for more details. ([#79992](https://github.com/kubernetes/kubernetes/pull/79992), [@deads2k](https://github.com/deads2k))\n\n  Courtesy of SIG API Machinery",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79992",
-    "pr_number": 79992,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80007": {
-    "commit": "ab960c612c4ca804fac2548a0f72bfd00761860c",
-    "text": "SubjectAccessReview requests sent for RBAC escalation, impersonation, and pod security policy authorization checks now populate the version attribute.",
-    "markdown": "SubjectAccessReview requests sent for RBAC escalation, impersonation, and pod security policy authorization checks now populate the version attribute. ([#80007](https://github.com/kubernetes/kubernetes/pull/80007), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80007",
-    "pr_number": 80007,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "auth"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "80017": {
-    "commit": "7e4a6ca3f77fe8cb33a788fee98a148327310dfd",
-    "text": "hyperkube: the `--make-symlinks` flag, deprecated in v1.14, has been removed.",
-    "markdown": "hyperkube: the `--make-symlinks` flag, deprecated in v1.14, has been removed. ([#80017](https://github.com/kubernetes/kubernetes/pull/80017), [@Pothulapati](https://github.com/Pothulapati))\n\n  Courtesy of SIG Release",
-    "author": "Pothulapati",
-    "author_url": "https://github.com/Pothulapati",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80017",
-    "pr_number": 80017,
-    "kinds": ["cleanup"],
-    "sigs": ["release"],
-    "action_required": true,
-    "release_version": "1.16.0"
-  },
-  "80024": {
-    "commit": "65fc256cc3ddd279849444c03c792d98d58f34ff",
-    "text": "kubeadm: fall back to client version in case of certain HTTP errors",
-    "markdown": "kubeadm: fall back to client version in case of certain HTTP errors ([#80024](https://github.com/kubernetes/kubernetes/pull/80024), [@RainbowMango](https://github.com/RainbowMango))",
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80024",
-    "pr_number": 80024,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "80025": {
-    "commit": "ae676161105fb88616850f9596021719df6d0106",
-    "text": "kubeadm: support fetching configuration from the original cluster for 'upgrade diff'",
-    "markdown": "kubeadm: support fetching configuration from the original cluster for 'upgrade diff' ([#80025](https://github.com/kubernetes/kubernetes/pull/80025), [@SataQiu](https://github.com/SataQiu))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80025",
-    "pr_number": 80025,
-    "areas": ["kubeadm"],
-    "kinds": ["bug", "cleanup", "feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80037": {
-    "commit": "aaad86c2105bfbf98f26745bf8262ab8c357e8d3",
-    "text": "etcd migration image no longer supports etcd2 version.",
-    "markdown": "etcd migration image no longer supports etcd2 version. ([#80037](https://github.com/kubernetes/kubernetes/pull/80037), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80037",
-    "pr_number": 80037,
-    "areas": ["code-organization", "dependency"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cluster-lifecycle"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "80043": {
-    "commit": "676417ec7bacca7b277aa2504667cd2c11633af0",
-    "text": "kubelet's --containerized flag was deprecated in 1.14. This flag is removed in 1.16.",
-    "markdown": "kubelet's --containerized flag was deprecated in 1.14. This flag is removed in 1.16. ([#80043](https://github.com/kubernetes/kubernetes/pull/80043), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80043",
-    "pr_number": 80043,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.16.0"
-  },
-  "80099": {
-    "commit": "844a0c9a10cd52367ec7c05f5a8c176535dcef38",
-    "text": "Cinder and ScaleIO volume providers have been deprecated and will be removed in a future release.",
-    "markdown": "Cinder and ScaleIO volume providers have been deprecated and will be removed in a future release. ([#80099](https://github.com/kubernetes/kubernetes/pull/80099), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80099",
-    "pr_number": 80099,
-    "kinds": ["cleanup"],
-    "sigs": ["storage"],
-    "release_version": "1.16.0"
-  },
-  "80100": {
-    "commit": "87b744715ec6952c45d04253dc7b63fc3cfe1ddc",
-    "text": "Increase log level for graceful termination to v=5",
-    "markdown": "Increase log level for graceful termination to v=5 ([#80100](https://github.com/kubernetes/kubernetes/pull/80100), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80100",
-    "pr_number": 80100,
-    "areas": ["ipvs"],
-    "kinds": ["cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.16.0"
-  },
-  "80115": {
-    "commit": "ab8506fb9b57671da54c9104cc080370592c0f80",
-    "text": "Add --kubernetes-version to \"kubeadm init phase certs ca\" and \"kubeadm init phase kubeconfig\"",
-    "markdown": "Add --kubernetes-version to \"kubeadm init phase certs ca\" and \"kubeadm init phase kubeconfig\" ([#80115](https://github.com/kubernetes/kubernetes/pull/80115), [@gyuho](https://github.com/gyuho))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "gyuho",
-    "author_url": "https://github.com/gyuho",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80115",
-    "pr_number": 80115,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80142": {
-    "commit": "fe0404171c9e33daf1aa0798dbeae0f40df1e3a0",
-    "text": "Remove deprecated github.com/kardianos/osext dependency",
-    "markdown": "Remove deprecated github.com/kardianos/osext dependency ([#80142](https://github.com/kubernetes/kubernetes/pull/80142), [@loqutus](https://github.com/loqutus))",
-    "author": "loqutus",
-    "author_url": "https://github.com/loqutus",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80142",
-    "pr_number": 80142,
-    "areas": ["dependency", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["testing"],
-    "release_version": "1.16.0"
-  },
-  "80145": {
-    "commit": "768acf600943442d517d16fdaaa786032099c995",
-    "text": "kubeadm ClusterConfiguration now supports featureGates: IPv6DualStack: true",
-    "markdown": "kubeadm ClusterConfiguration now supports featureGates: IPv6DualStack: true ([#80145](https://github.com/kubernetes/kubernetes/pull/80145), [@Arvinderpal](https://github.com/Arvinderpal))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "Arvinderpal",
-    "author_url": "https://github.com/Arvinderpal",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80145",
-    "pr_number": 80145,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80157": {
-    "commit": "635b83c5f4065c5528dea2697c605cda969dd2cc",
-    "text": "Fix a bug that ListOptions.AllowWatchBookmarks wasn't propagating correctly in kube-apiserver.",
-    "markdown": "Fix a bug that ListOptions.AllowWatchBookmarks wasn't propagating correctly in kube-apiserver. ([#80157](https://github.com/kubernetes/kubernetes/pull/80157), [@wojtek-t](https://github.com/wojtek-t))",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80157",
-    "pr_number": 80157,
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery", "scalability"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "80173": {
-    "commit": "ab266f725ff576b067750b2ae36f41b08f8cb45a",
-    "text": "make node lease renew interval more heuristic based on node-status-update-frequency in kubelet",
-    "markdown": "make node lease renew interval more heuristic based on node-status-update-frequency in kubelet ([#80173](https://github.com/kubernetes/kubernetes/pull/80173), [@gaorong](https://github.com/gaorong))",
-    "author": "gaorong",
-    "author_url": "https://github.com/gaorong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80173",
-    "pr_number": 80173,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.16.0"
-  },
-  "80176": {
-    "commit": "04c9bd9bfa420d369e131f7bc80eb3e212284bbe",
-    "text": "cpuUsageNanoCores is now reported in the Kubelet summary API on Windows nodes",
-    "markdown": "cpuUsageNanoCores is now reported in the Kubelet summary API on Windows nodes ([#80176](https://github.com/kubernetes/kubernetes/pull/80176), [@liyanhui1228](https://github.com/liyanhui1228))",
-    "author": "liyanhui1228",
-    "author_url": "https://github.com/liyanhui1228",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80176",
-    "pr_number": 80176,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "windows"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "80191": {
-    "commit": "835552ecb6626e9751e52d22c6afbc3e4aa1231f",
-    "text": "Pass-through volume MountOptions to global mount (NodeStageVolume) on the node for CSI",
-    "markdown": "Pass-through volume MountOptions to global mount (NodeStageVolume) on the node for CSI ([#80191](https://github.com/kubernetes/kubernetes/pull/80191), [@davidz627](https://github.com/davidz627))",
-    "author": "davidz627",
-    "author_url": "https://github.com/davidz627",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80191",
-    "pr_number": 80191,
-    "areas": ["test"],
-    "kinds": ["bug", "failing-test"],
-    "sigs": ["storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "80231": {
-    "commit": "c981c65c9013306d9a65aedbd30479b932e427b2",
-    "text": "The `AdmissionReview` API sent to and received from admission webhooks has been promoted to `admission.k8s.io/v1`. Webhooks can specify a preference for receiving `v1` AdmissionReview objects with `admissionReviewVersions: [\"v1\",\"v1beta1\"]`, and must respond with an API object in the same `apiVersion` they are sent. When webhooks use `admission.k8s.io/v1`, the following additional validation is performed on their responses:\n- `response.patch` and `response.patchType` are not permitted from validating admission webhooks\n- `apiVersion: \"admission.k8s.io/v1\"` is required\n- `kind: \"AdmissionReview\"` is required\n- `response.uid: \"\u003cvalue of request.uid\u003e\"` is required\n- `response.patchType: \"JSONPatch\"` is required (if `response.patch` is set)",
-    "markdown": "The `AdmissionReview` API sent to and received from admission webhooks has been promoted to `admission.k8s.io/v1`. Webhooks can specify a preference for receiving `v1` AdmissionReview objects with `admissionReviewVersions: [\"v1\",\"v1beta1\"]`, and must respond with an API object in the same `apiVersion` they are sent. When webhooks use `admission.k8s.io/v1`, the following additional validation is performed on their responses:\n  - `response.patch` and `response.patchType` are not permitted from validating admission webhooks\n  - `apiVersion: \"admission.k8s.io/v1\"` is required\n  - `kind: \"AdmissionReview\"` is required\n  - `response.uid: \"\u003cvalue of request.uid\u003e\"` is required\n  - `response.patchType: \"JSONPatch\"` is required (if `response.patch` is set) ([#80231](https://github.com/kubernetes/kubernetes/pull/80231), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG CLI, SIG Node, and SIG Testing",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80231",
-    "pr_number": 80231,
-    "areas": ["apiserver", "kubectl", "kubelet", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "auth", "cli", "node", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80238": {
-    "commit": "92a320aeb632d6525acafd1869005a4d05560de6",
-    "text": "Introduce `node.kubernetes.io/exclude-balancer` and `node.kubernetes.io/exclude-disruption` labels in alpha to prevent cluster deployers from being dependent on the optional `node-role` labels which not all clusters may provide.",
-    "markdown": "Introduce `node.kubernetes.io/exclude-balancer` and `node.kubernetes.io/exclude-disruption` labels in alpha to prevent cluster deployers from being dependent on the optional `node-role` labels which not all clusters may provide. ([#80238](https://github.com/kubernetes/kubernetes/pull/80238), [@smarterclayton](https://github.com/smarterclayton))\n\n  Courtesy of SIG Apps, SIG Network, and SIG Testing",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80238",
-    "pr_number": 80238,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["feature"],
-    "sigs": ["apps", "network", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80253": {
-    "commit": "18d7f8839d9330ab64a4e689d64ad7a79fe4e8a9",
-    "text": "`framework.ExpectNoError` no longer logs the error and instead relies on using the new `log.Fail` as Gomega fail handler.",
-    "markdown": "`framework.ExpectNoError` no longer logs the error and instead relies on using the new `log.Fail` as Gomega fail handler. ([#80253](https://github.com/kubernetes/kubernetes/pull/80253), [@pohly](https://github.com/pohly))\n\n  Courtesy of SIG Testing",
-    "author": "pohly",
-    "author_url": "https://github.com/pohly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80253",
-    "pr_number": 80253,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["feature"],
-    "sigs": ["testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80266": {
-    "commit": "5de41340ce3917ab948573f6358b10512a0333ce",
-    "text": "container images tar files for 'amd64' will now contain the architecture in the RepoTags manifest.json section.\nIf you are using docker manifests there are not visible changes.",
-    "markdown": "container images tar files for 'amd64' will now contain the architecture in the RepoTags manifest.json section.\n  If you are using docker manifests there are not visible changes. ([#80266](https://github.com/kubernetes/kubernetes/pull/80266), [@javier-b-perez](https://github.com/javier-b-perez))\n\n  Courtesy of SIG Release",
-    "author": "javier-b-perez",
-    "author_url": "https://github.com/javier-b-perez",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80266",
-    "pr_number": 80266,
-    "kinds": ["bug"],
-    "sigs": ["release"],
-    "action_required": true,
-    "release_version": "1.16.0"
-  },
-  "80277": {
-    "commit": "49f6510d9af15e4cf0b70fc2011f5db956258bbe",
-    "text": "Revert \"scheduler.alpha.kubernetes.io/critical-pod annotation is removed. Pod priority (spec.priorityClassName) should be used instead to mark pods as critical. Action required!\"",
-    "markdown": "Revert \"scheduler.alpha.kubernetes.io/critical-pod annotation is removed. Pod priority (spec.priorityClassName) should be used instead to mark pods as critical. Action required!\" ([#80277](https://github.com/kubernetes/kubernetes/pull/80277), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Apps, SIG Cluster Lifecycle, SIG Instrumentation, SIG Node, SIG Scheduling, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80277",
-    "pr_number": 80277,
-    "areas": ["kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "cluster-lifecycle", "instrumentation", "node", "scheduling", "testing"],
-    "action_required": true,
-    "release_version": "1.16.0"
-  },
-  "80294": {
-    "commit": "304e2c247ae7566279fdba73895a24221ed583d8",
-    "text": "Passing an invalid policy name in the `--cpu-manager-policy` flag will now cause the kubelet to fail instead of simply ignoring the flag and running the `cpumanager`s default policy instead.",
-    "markdown": "Passing an invalid policy name in the `--cpu-manager-policy` flag will now cause the kubelet to fail instead of simply ignoring the flag and running the `cpumanager`s default policy instead. ([#80294](https://github.com/kubernetes/kubernetes/pull/80294), [@klueska](https://github.com/klueska))",
-    "author": "klueska",
-    "author_url": "https://github.com/klueska",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80294",
-    "pr_number": 80294,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.16.0"
-  },
-  "80337": {
-    "commit": "bd6f58cfcbab921d0f9431ae6cc5ad2cc0eb6e2f",
-    "text": "Remove deprecated \"include-uninitialized\" flag. action required",
-    "markdown": "Remove deprecated \"include-uninitialized\" flag. action required ([#80337](https://github.com/kubernetes/kubernetes/pull/80337), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG CLI",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80337",
-    "pr_number": 80337,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "action_required": true,
-    "release_version": "1.16.0"
-  },
-  "80342": {
-    "commit": "273e9262bb1de2e31a57065ea082dee8585e2ed1",
-    "text": "scheduler.alpha.kubernetes.io/critical-pod annotation is removed. Pod priority (spec.priorityClassName) should be used instead to mark pods as critical. Action required!",
-    "markdown": "scheduler.alpha.kubernetes.io/critical-pod annotation is removed. Pod priority (spec.priorityClassName) should be used instead to mark pods as critical. Action required! ([#80342](https://github.com/kubernetes/kubernetes/pull/80342), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Cluster Lifecycle, SIG Instrumentation, SIG Node, SIG Scheduling, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80342",
-    "pr_number": 80342,
-    "areas": ["apiserver", "kubelet", "provider/gcp", "test"],
-    "kinds": ["cleanup"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "cluster-lifecycle",
-      "instrumentation",
-      "node",
-      "scheduling",
-      "testing"
-    ],
-    "action_required": true,
-    "release_version": "1.16.0"
-  },
-  "80361": {
-    "commit": "69a34f6a6f67de47cb9b72b6ac98e089d301beb3",
-    "text": "Remove GetReference() and GetPartialReference() function from pkg/api/ref, as the same function exists also in staging/src/k8s.io/client-go/tools/ref",
-    "markdown": "Remove GetReference() and GetPartialReference() function from pkg/api/ref, as the same function exists also in staging/src/k8s.io/client-go/tools/ref ([#80361](https://github.com/kubernetes/kubernetes/pull/80361), [@wojtek-t](https://github.com/wojtek-t))",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80361",
-    "pr_number": 80361,
-    "areas": ["code-organization"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.0"
-  },
-  "80368": {
-    "commit": "37651f1cef5ec5c7286ba409ee0fe298c4605d66",
-    "text": "Fixes a problem with the iptables proxy mode that could result in long delays\nupdating Service/Endpoints IPs in very large clusters on RHEL/CentOS 7.",
-    "markdown": "Fixes a problem with the iptables proxy mode that could result in long delays\n  updating Service/Endpoints IPs in very large clusters on RHEL/CentOS 7. ([#80368](https://github.com/kubernetes/kubernetes/pull/80368), [@danwinship](https://github.com/danwinship))",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80368",
-    "pr_number": 80368,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["network", "node", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "80369": {
-    "commit": "1fcd3d6baced5aeb3f958d0819afc71db96bcc23",
-    "text": "Errors from pod volume set up are now propagated as pod events.",
-    "markdown": "Errors from pod volume set up are now propagated as pod events. ([#80369](https://github.com/kubernetes/kubernetes/pull/80369), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80369",
-    "pr_number": 80369,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "storage"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "80376": {
-    "commit": "f6a70ef27163d1171412b6f97049968896b8d1f9",
-    "text": "Removed cadvisor metric labels `pod_name` and `container_name` to match instrumentation guidelines.\n\nany Prometheus queries that match `pod_name` and `container_name` labels (e.g. cadvisor or kubelet probe metrics) must be updated to use `pod` and `container` instead.",
-    "markdown": "Removed cadvisor metric labels `pod_name` and `container_name` to match instrumentation guidelines.\n  \n  any Prometheus queries that match `pod_name` and `container_name` labels (e.g. cadvisor or kubelet probe metrics) must be updated to use `pod` and `container` instead. ([#80376](https://github.com/kubernetes/kubernetes/pull/80376), [@ehashman](https://github.com/ehashman))\n\n  Courtesy of SIG Instrumentation, SIG Node, and SIG Testing",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/28d0f41a30b32bceb01e0d4ae338f91ede195005/keps/sig-instrumentation/20181106-kubernetes-metrics-overhaul.md#cadvisor-instrumentation-changes",
-        "type": "KEP"
-      }
-    ],
-    "author": "ehashman",
-    "author_url": "https://github.com/ehashman",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80376",
-    "pr_number": 80376,
-    "areas": ["kubelet", "test"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["instrumentation", "node", "testing"],
-    "action_required": true,
-    "release_version": "1.16.0"
-  },
-  "80383": {
-    "commit": "fb46ec845528dae0796a581583fd4fcdc764c104",
-    "text": "Implement normalize plugin extension point for the scheduler framework.",
-    "markdown": "Implement normalize plugin extension point for the scheduler framework. ([#80383](https://github.com/kubernetes/kubernetes/pull/80383), [@liu-cong](https://github.com/liu-cong))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Architecture, SIG Auth, SIG Autoscaling, SIG CLI, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Instrumentation, SIG Network, SIG Node, SIG Release, SIG Scheduling, SIG Storage, and SIG Testing",
-    "author": "liu-cong",
-    "author_url": "https://github.com/liu-cong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80383",
-    "pr_number": 80383,
-    "areas": [
-      "apiserver",
-      "cloudprovider",
-      "code-generation",
-      "conformance",
-      "dependency",
-      "e2e-test-framework",
-      "kubeadm",
-      "kubectl",
-      "kubelet",
-      "release-eng",
-      "test"
-    ],
-    "kinds": ["api-change", "feature"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "architecture",
-      "auth",
-      "autoscaling",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "network",
-      "node",
-      "release",
-      "scheduling",
-      "storage",
-      "testing"
-    ],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80419": {
-    "commit": "c08a88a2d14c26a5376149c6a3301db410cd5ca4",
-    "text": "Fix retry issues when the nodes are under deleting on Azure",
-    "markdown": "Fix retry issues when the nodes are under deleting on Azure ([#80419](https://github.com/kubernetes/kubernetes/pull/80419), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80419",
-    "pr_number": 80419,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.0"
-  },
-  "80421": {
-    "commit": "10005d2e1e1425904f8c7bf5615e730fb0fea7c9",
-    "text": "Kibana has been slightly revamped/improved in the latest version",
-    "markdown": "Kibana has been slightly revamped/improved in the latest version ([#80421](https://github.com/kubernetes/kubernetes/pull/80421), [@lostick](https://github.com/lostick))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Instrumentation",
-    "author": "lostick",
-    "author_url": "https://github.com/lostick",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80421",
-    "pr_number": 80421,
-    "areas": ["dependency"],
-    "kinds": ["bug", "feature"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80429": {
-    "commit": "f18833ee582d9f3bbe57e5f7c97f933c455640dc",
-    "text": "kubelet: change node-lease-renew-interval to 0.25 of lease-renew-duration",
-    "markdown": "kubelet: change node-lease-renew-interval to 0.25 of lease-renew-duration ([#80429](https://github.com/kubernetes/kubernetes/pull/80429), [@gaorong](https://github.com/gaorong))\n\n  Courtesy of SIG Architecture, and SIG Node",
-    "author": "gaorong",
-    "author_url": "https://github.com/gaorong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80429",
-    "pr_number": 80429,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["architecture", "node"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80436": {
-    "commit": "08f9f2b58a6aa4924e9083f4e04735858e1c68db",
-    "text": "Fix CVE-2019-11249: Incomplete fixes for CVE-2019-1002101 and CVE-2019-11246, kubectl cp potential directory traversal",
-    "markdown": "Fix CVE-2019-11249: Incomplete fixes for CVE-2019-1002101 and CVE-2019-11246, kubectl cp potential directory traversal ([#80436](https://github.com/kubernetes/kubernetes/pull/80436), [@M00nF1sh](https://github.com/M00nF1sh))",
-    "author": "M00nF1sh",
-    "author_url": "https://github.com/M00nF1sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80436",
-    "pr_number": 80436,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.16.0"
-  },
-  "80443": {
-    "commit": "030274eba30668968ecfb43cf5ef3dfc642c3f4d",
-    "text": "Add CSI Migration Shim for VerifyVolumesAreAttached and BulkVolumeVerify",
-    "markdown": "Add CSI Migration Shim for VerifyVolumesAreAttached and BulkVolumeVerify ([#80443](https://github.com/kubernetes/kubernetes/pull/80443), [@davidz627](https://github.com/davidz627))\n\n  Courtesy of SIG Storage",
-    "author": "davidz627",
-    "author_url": "https://github.com/davidz627",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80443",
-    "pr_number": 80443,
-    "kinds": ["feature"],
-    "sigs": ["storage"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80477": {
-    "commit": "ff1e1127e27d4a144de98082c66800f2300592b0",
-    "text": "Added a metric 'apiserver_watch_events_sizes' that can be used to estimate sizes of watch events in the system.",
-    "markdown": "Added a metric 'apiserver_watch_events_sizes' that can be used to estimate sizes of watch events in the system. ([#80477](https://github.com/kubernetes/kubernetes/pull/80477), [@mborsz](https://github.com/mborsz))\n\n  Courtesy of SIG API Machinery",
-    "author": "mborsz",
-    "author_url": "https://github.com/mborsz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80477",
-    "pr_number": 80477,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80485": {
-    "commit": "3e7e12dac83c90cc9fee1c50cb9d4460a9e14d59",
-    "text": "Azure supports IPv6 only on ELB not ILB. The cloud provider will return an error if the service is internal and is IPv6.\n\nNotes on LB name:\nto ensure backword and forward compat:\n- SingleStack -v4 (pre v1.16) =\u003e BackendPool name == clusterName\n- SingleStack -v6 =\u003e BackendPool name == clusterName (all cluster bootstrap uses this name)\nDualStack:\n=\u003e IPv4 BackendPool name == clusterName\n=\u003e IPv6 BackendPool name == \u003cclusterName\u003e-IPv6\nThis result into:\n- clusters moving from IPv4 to duakstack will require no changes\n- clusters moving from IPv6 (while not seen in the wild, we can not rule out thier existance) to dualstack will require deleting backend pools (the reconciler will take care of creating correct backendpools)",
-    "markdown": "Azure supports IPv6 only on ELB not ILB. The cloud provider will return an error if the service is internal and is IPv6.\n  \n  Notes on LB name:\n  to ensure backword and forward compat:\n  - SingleStack -v4 (pre v1.16) =\u003e BackendPool name == clusterName\n  - SingleStack -v6 =\u003e BackendPool name == clusterName (all cluster bootstrap uses this name)\n  DualStack:\n  =\u003e IPv4 BackendPool name == clusterName\n  =\u003e IPv6 BackendPool name == \u003cclusterName\u003e-IPv6\n  This result into:\n  - clusters moving from IPv4 to duakstack will require no changes\n  - clusters moving from IPv6 (while not seen in the wild, we can not rule out thier existance) to dualstack will require deleting backend pools (the reconciler will take care of creating correct backendpools) ([#80485](https://github.com/kubernetes/kubernetes/pull/80485), [@khenidak](https://github.com/khenidak))",
-    "author": "khenidak",
-    "author_url": "https://github.com/khenidak",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80485",
-    "pr_number": 80485,
-    "areas": [
-      "apiserver",
-      "cloudprovider",
-      "kubectl",
-      "kubelet",
-      "provider/azure",
-      "release-eng",
-      "test"
-    ],
-    "kinds": ["api-change"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "cli",
-      "cloud-provider",
-      "network",
-      "node",
-      "release",
-      "testing"
-    ],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "80491": {
-    "commit": "806ced34a42111eff1bfafb46b4838db0a9a3bfc",
-    "text": "Kubelet considers all static pods as critical. Static pods pass kubelet admission even if a node does not have enough resources. Users must ensure that they account for resources when creating static pods.",
-    "markdown": "Kubelet considers all static pods as critical. Static pods pass kubelet admission even if a node does not have enough resources. Users must ensure that they account for resources when creating static pods. ([#80491](https://github.com/kubernetes/kubernetes/pull/80491), [@hpandeycodeit](https://github.com/hpandeycodeit))",
-    "author": "hpandeycodeit",
-    "author_url": "https://github.com/hpandeycodeit",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80491",
-    "pr_number": 80491,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "80495": {
-    "commit": "de73f62336c9dcc80ef87a46f8c0127ddf6d1c6c",
-    "text": "Update portworx plugin dependency on libopenstorage/openstorage to v1.0.0",
-    "markdown": "Update portworx plugin dependency on libopenstorage/openstorage to v1.0.0 ([#80495](https://github.com/kubernetes/kubernetes/pull/80495), [@adityadani](https://github.com/adityadani))",
-    "author": "adityadani",
-    "author_url": "https://github.com/adityadani",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80495",
-    "pr_number": 80495,
-    "areas": ["code-organization", "dependency"],
-    "kinds": ["cleanup"],
-    "sigs": ["storage"],
-    "release_version": "1.16.0"
-  },
-  "80509": {
-    "commit": "44de947f0f86c0b2558b4ef0872801bd258a5949",
-    "text": "New flag --endpoint-updates-batch-period in kube-controller-manager can be used to reduce number of endpoints updates generated by pod changes.",
-    "markdown": "New flag --endpoint-updates-batch-period in kube-controller-manager can be used to reduce number of endpoints updates generated by pod changes. ([#80509](https://github.com/kubernetes/kubernetes/pull/80509), [@mborsz](https://github.com/mborsz))\n\n  Courtesy of SIG Apps, and SIG Network",
-    "author": "mborsz",
-    "author_url": "https://github.com/mborsz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80509",
-    "pr_number": 80509,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["apps", "network"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80518": {
-    "commit": "1a95199311a2ad72cf6f85de19f25e616869e207",
-    "text": "Fixed detachment of deleted volumes on OpenStack / Cinder.",
-    "markdown": "Fixed detachment of deleted volumes on OpenStack / Cinder. ([#80518](https://github.com/kubernetes/kubernetes/pull/80518), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80518",
-    "pr_number": 80518,
-    "areas": ["cloudprovider", "provider/openstack"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "80529": {
-    "commit": "feb40b844c12e0b7e528bbcdaa41f218b3f5b881",
-    "text": "kubeadm: fixed ignoring errors when pulling control plane images",
-    "markdown": "kubeadm: fixed ignoring errors when pulling control plane images ([#80529](https://github.com/kubernetes/kubernetes/pull/80529), [@bart0sh](https://github.com/bart0sh))",
-    "author": "bart0sh",
-    "author_url": "https://github.com/bart0sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80529",
-    "pr_number": 80529,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "80531": {
-    "commit": "5bfa3664fb8ad22a784a8e0ce4107dd4146b029e",
-    "text": "In order to enable dual-stack support within kubeadm and kubernetes components, as part of the init config file, the user should set feature-gate IPv6DualStack=true in the ClusterConfiguration. Additionally, for each worker node, the user should set the feature-gate for kubelet using either nodeRegistration.kubeletExtraArgs or  KUBELET_EXTRA_ARGS.",
-    "markdown": "In order to enable dual-stack support within kubeadm and kubernetes components, as part of the init config file, the user should set feature-gate IPv6DualStack=true in the ClusterConfiguration. Additionally, for each worker node, the user should set the feature-gate for kubelet using either nodeRegistration.kubeletExtraArgs or  KUBELET_EXTRA_ARGS. ([#80531](https://github.com/kubernetes/kubernetes/pull/80531), [@Arvinderpal](https://github.com/Arvinderpal))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "Arvinderpal",
-    "author_url": "https://github.com/Arvinderpal",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80531",
-    "pr_number": 80531,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80568": {
-    "commit": "087aafc18d4631728187f014a487b30d2a0986d3",
-    "text": "a CSI driver that supports ephemeral inline volumes must explicitly declare that by providing a CSIDriver object where the new \"mode\" field is set to \"ephemeral\" or \"persistent+ephemeral\"",
-    "markdown": "a CSI driver that supports ephemeral inline volumes must explicitly declare that by providing a CSIDriver object where the new \"mode\" field is set to \"ephemeral\" or \"persistent+ephemeral\" ([#80568](https://github.com/kubernetes/kubernetes/pull/80568), [@pohly](https://github.com/pohly))",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/20190122-csi-inline-volumes.md#driver-mode",
-        "type": "KEP"
-      }
-    ],
-    "author": "pohly",
-    "author_url": "https://github.com/pohly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80568",
-    "pr_number": 80568,
-    "areas": ["test"],
-    "kinds": ["api-change"],
-    "sigs": ["storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "80573": {
-    "commit": "8f1eeacb78aa99c5e8d93998912cc94cf8215c44",
-    "text": "kubeadm: produce errors if they occur when resetting cluster status for a control-plane node",
-    "markdown": "kubeadm: produce errors if they occur when resetting cluster status for a control-plane node ([#80573](https://github.com/kubernetes/kubernetes/pull/80573), [@bart0sh](https://github.com/bart0sh))",
-    "author": "bart0sh",
-    "author_url": "https://github.com/bart0sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80573",
-    "pr_number": 80573,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "80575": {
-    "commit": "a1f4c2ed21ea008c59934a9c6a06fac3da72f963",
-    "text": "Fixed a bug in kube-addon-manager's leader election logic that made all replicas active.",
-    "markdown": "Fixed a bug in kube-addon-manager's leader election logic that made all replicas active. ([#80575](https://github.com/kubernetes/kubernetes/pull/80575), [@mborsz](https://github.com/mborsz))",
-    "author": "mborsz",
-    "author_url": "https://github.com/mborsz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80575",
-    "pr_number": 80575,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "80591": {
-    "commit": "0775e6b2db031f84b8e7116547b8b5cbe0cfd51a",
-    "text": "Fixes problems with connecting to services on localhost on some systems; in particular, DNS queries to systemd-resolved on Ubuntu.",
-    "markdown": "Fixes problems with connecting to services on localhost on some systems; in particular, DNS queries to systemd-resolved on Ubuntu. ([#80591](https://github.com/kubernetes/kubernetes/pull/80591), [@danwinship](https://github.com/danwinship))",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80591",
-    "pr_number": 80591,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["network", "node"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "80624": {
-    "commit": "78915a10949e74feb77586f71ee035292462ea56",
-    "text": "When a load balancer type service is created in a k8s cluster that is backed by Azure Standard Load Balancer, the corresponding load balancer rule added in the Azure Standard Load Balancer would now have the \"EnableTcpReset\" property set to true.",
-    "markdown": "When a load balancer type service is created in a k8s cluster that is backed by Azure Standard Load Balancer, the corresponding load balancer rule added in the Azure Standard Load Balancer would now have the \"EnableTcpReset\" property set to true. ([#80624](https://github.com/kubernetes/kubernetes/pull/80624), [@xuto2](https://github.com/xuto2))",
-    "author": "xuto2",
-    "author_url": "https://github.com/xuto2",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80624",
-    "pr_number": 80624,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.0"
-  },
-  "80648": {
-    "commit": "9bae1bc56804db4905abebcd408e0f02e199ab93",
-    "text": "Fix error handling and potential go null pointer exception in kubeadm upgrade diff",
-    "markdown": "Fix error handling and potential go null pointer exception in kubeadm upgrade diff ([#80648](https://github.com/kubernetes/kubernetes/pull/80648), [@odinuge](https://github.com/odinuge))",
-    "author": "odinuge",
-    "author_url": "https://github.com/odinuge",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80648",
-    "pr_number": 80648,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "80675": {
-    "commit": "000932d5844c2d2e01f99872e3df268cf7c25d14",
-    "text": "Fix error in `kubeadm join --discovery-file` when using discovery files with embedded credentials",
-    "markdown": "Fix error in `kubeadm join --discovery-file` when using discovery files with embedded credentials ([#80675](https://github.com/kubernetes/kubernetes/pull/80675), [@fabriziopandini](https://github.com/fabriziopandini))",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80675",
-    "pr_number": 80675,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "80676": {
-    "commit": "82a252ad746d0230519ee2da78b6b5fef4896fde",
-    "text": "kubeadm now deletes the bootstrap-kubelet.conf file after TLS bootstrap\nUser relying on bootstrap-kubelet.conf should switch to kubelet.conf that contains node credentials",
-    "markdown": "kubeadm now deletes the bootstrap-kubelet.conf file after TLS bootstrap\n  User relying on bootstrap-kubelet.conf should switch to kubelet.conf that contains node credentials ([#80676](https://github.com/kubernetes/kubernetes/pull/80676), [@fabriziopandini](https://github.com/fabriziopandini))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80676",
-    "pr_number": 80676,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "action_required": true,
-    "release_version": "1.16.0"
-  },
-  "80696": {
-    "commit": "9aa76b2c8aef62e1ad352451444060116c052989",
-    "text": "Add a helper function to decode scheduler plugin args.",
-    "markdown": "Add a helper function to decode scheduler plugin args. ([#80696](https://github.com/kubernetes/kubernetes/pull/80696), [@hex108](https://github.com/hex108))",
-    "author": "hex108",
-    "author_url": "https://github.com/hex108",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80696",
-    "pr_number": 80696,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.16.0"
-  },
-  "80699": {
-    "commit": "182ae7b9a47f14f55c3fed5f8f948ace647263e9",
-    "text": "Add a `Patch` method to `ScaleInterface`",
-    "markdown": "Add a `Patch` method to `ScaleInterface` ([#80699](https://github.com/kubernetes/kubernetes/pull/80699), [@knight42](https://github.com/knight42))\n\n  Courtesy of SIG API Machinery",
-    "author": "knight42",
-    "author_url": "https://github.com/knight42",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80699",
-    "pr_number": 80699,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80703": {
-    "commit": "5413252e47fcb9b05059bc2061786e25575b6cd3",
-    "text": "Fix public IP not found issues for VMSS nodes",
-    "markdown": "Fix public IP not found issues for VMSS nodes ([#80703](https://github.com/kubernetes/kubernetes/pull/80703), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80703",
-    "pr_number": 80703,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.0"
-  },
-  "80726": {
-    "commit": "b9c6f559bd0137c87714feea950d69f1090d14c9",
-    "text": "Node-Problem-Detector v0.7.1 is used on GCI.",
-    "markdown": "Node-Problem-Detector v0.7.1 is used on GCI. ([#80726](https://github.com/kubernetes/kubernetes/pull/80726), [@wangzhen127](https://github.com/wangzhen127))\n\n  Courtesy of SIG Cluster Lifecycle, SIG Node, and SIG Testing",
-    "author": "wangzhen127",
-    "author_url": "https://github.com/wangzhen127",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80726",
-    "pr_number": 80726,
-    "areas": ["dependency", "test"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "node", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80750": {
-    "commit": "2abc859540cca6124828614facb491356964fc1f",
-    "text": "Fix CVE-2019-11247: API server allows access to custom resources via wrong scope",
-    "markdown": "Fix CVE-2019-11247: API server allows access to custom resources via wrong scope ([#80750](https://github.com/kubernetes/kubernetes/pull/80750), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80750",
-    "pr_number": 80750,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.0"
-  },
-  "80764": {
-    "commit": "56d7920e8fdefe9505b4e6ce4798619940ca0a28",
-    "text": "Determine system model to get credentials for windows nodes.",
-    "markdown": "Determine system model to get credentials for windows nodes. ([#80764](https://github.com/kubernetes/kubernetes/pull/80764), [@liyanhui1228](https://github.com/liyanhui1228))",
-    "author": "liyanhui1228",
-    "author_url": "https://github.com/liyanhui1228",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80764",
-    "pr_number": 80764,
-    "kinds": ["bug"],
-    "release_version": "1.16.0"
-  },
-  "80773": {
-    "commit": "3dbedefb80a54e39e6496853a8ac6f98d7a08757",
-    "text": "Fixed a bug with the openAPI definition for io.k8s.apimachinery.pkg.runtime.RawExtension, which previously required a field \"raw\" to be specified",
-    "markdown": "Fixed a bug with the openAPI definition for io.k8s.apimachinery.pkg.runtime.RawExtension, which previously required a field \"raw\" to be specified ([#80773](https://github.com/kubernetes/kubernetes/pull/80773), [@jennybuckley](https://github.com/jennybuckley))",
-    "author": "jennybuckley",
-    "author_url": "https://github.com/jennybuckley",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80773",
-    "pr_number": 80773,
-    "areas": ["code-generation"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.0"
-  },
-  "80798": {
-    "commit": "3758426884e3c82cbd99c72e8015f4396f21fde2",
-    "text": "kubeadm: avoid double deletion of the upgrade prepull DaemonSet",
-    "markdown": "kubeadm: avoid double deletion of the upgrade prepull DaemonSet ([#80798](https://github.com/kubernetes/kubernetes/pull/80798), [@xlgao-zju](https://github.com/xlgao-zju))",
-    "author": "xlgao-zju",
-    "author_url": "https://github.com/xlgao-zju",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80798",
-    "pr_number": 80798,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "80802": {
-    "commit": "c91761da0d854aa842ea55ba4d7c6cbc8d675892",
-    "text": "Fixed the bash completion error with override flags.",
-    "markdown": "Fixed the bash completion error with override flags. ([#80802](https://github.com/kubernetes/kubernetes/pull/80802), [@dtaniwaki](https://github.com/dtaniwaki))",
-    "author": "dtaniwaki",
-    "author_url": "https://github.com/dtaniwaki",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80802",
-    "pr_number": 80802,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.16.0"
-  },
-  "80804": {
-    "commit": "547617a523d9a1e6db9be68258303d5adb416e78",
-    "text": "\"kubeadm join\" fails if file-based discovery is too long, with a default timeout of 5 minutes.",
-    "markdown": "\"kubeadm join\" fails if file-based discovery is too long, with a default timeout of 5 minutes. ([#80804](https://github.com/kubernetes/kubernetes/pull/80804), [@olivierlemasle](https://github.com/olivierlemasle))",
-    "author": "olivierlemasle",
-    "author_url": "https://github.com/olivierlemasle",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80804",
-    "pr_number": 80804,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "80825": {
-    "commit": "2f91fcc3fd85470128981b2051152f560dca4cf0",
-    "text": "Add scheduling support for RuntimeClasses. RuntimeClasses can now specify nodeSelector constraints \u0026 tolerations, which are merged into the PodSpec for pods using that RuntimeClass.",
-    "markdown": "Add scheduling support for RuntimeClasses. RuntimeClasses can now specify nodeSelector constraints \u0026 tolerations, which are merged into the PodSpec for pods using that RuntimeClass. ([#80825](https://github.com/kubernetes/kubernetes/pull/80825), [@tallclair](https://github.com/tallclair))",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/runtime-class-scheduling.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80825",
-    "pr_number": 80825,
-    "kinds": ["api-change"],
-    "sigs": ["node"],
-    "release_version": "1.16.0"
-  },
-  "80837": {
-    "commit": "16d9a659da541fc38110c112c6ae20a1056baf37",
-    "text": "Azure disks of shared kind will no longer fail if they do not contain skuname or \n    storageaccounttype.",
-    "markdown": "Azure disks of shared kind will no longer fail if they do not contain skuname or \n      storageaccounttype. ([#80837](https://github.com/kubernetes/kubernetes/pull/80837), [@rmweir](https://github.com/rmweir))",
-    "author": "rmweir",
-    "author_url": "https://github.com/rmweir",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80837",
-    "pr_number": 80837,
-    "areas": ["provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.16.0"
-  },
-  "80841": {
-    "commit": "9c22ff19663ffb43491152c41e2292325d021ef5",
-    "text": "enhance Azure cloud provider code to support both AAD and ADFS authentication.",
-    "markdown": "enhance Azure cloud provider code to support both AAD and ADFS authentication. ([#80841](https://github.com/kubernetes/kubernetes/pull/80841), [@rjaini](https://github.com/rjaini))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "rjaini",
-    "author_url": "https://github.com/rjaini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80841",
-    "pr_number": 80841,
-    "areas": ["cloudprovider"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80861": {
-    "commit": "231a6a7850d2bd4eb1ae85f87dbc9ca14c373589",
-    "text": "kube-addon-manager has been updated to v9.0.2 to fix a bug in leader election (https://github.com/kubernetes/kubernetes/pull/80575)",
-    "markdown": "kube-addon-manager has been updated to v9.0.2 to fix a bug in leader election (https://github.com/kubernetes/kubernetes/pull/80575) ([#80861](https://github.com/kubernetes/kubernetes/pull/80861), [@mborsz](https://github.com/mborsz))",
-    "author": "mborsz",
-    "author_url": "https://github.com/mborsz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80861",
-    "pr_number": 80861,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "80862": {
-    "commit": "564eff757cdd26a1322458876287298b8434b78b",
-    "text": "kubeadm: treat non-fatal errors as warnings when doing reset",
-    "markdown": "kubeadm: treat non-fatal errors as warnings when doing reset ([#80862](https://github.com/kubernetes/kubernetes/pull/80862), [@drpaneas](https://github.com/drpaneas))",
-    "author": "drpaneas",
-    "author_url": "https://github.com/drpaneas",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80862",
-    "pr_number": 80862,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "80867": {
-    "commit": "44f3aafc1a8ce9c1b5972cbfa12aa9b583fdfe7b",
-    "text": "This PR fixes an error when using external etcd but storing etcd certificates in the same folder and with the same name used by kubeadm for local etcd certificates; for an older version of kubeadm, the workaround is to avoid file name used by kubeadm for local etcd.",
-    "markdown": "This PR fixes an error when using external etcd but storing etcd certificates in the same folder and with the same name used by kubeadm for local etcd certificates; for an older version of kubeadm, the workaround is to avoid file name used by kubeadm for local etcd. ([#80867](https://github.com/kubernetes/kubernetes/pull/80867), [@fabriziopandini](https://github.com/fabriziopandini))",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80867",
-    "pr_number": 80867,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "80901": {
-    "commit": "1906650d9284ffebe97cb0abd17f8fbdb5c63b93",
-    "text": "Use a named array instead of a score array in normalizing-score phase.",
-    "markdown": "Use a named array instead of a score array in normalizing-score phase. ([#80901](https://github.com/kubernetes/kubernetes/pull/80901), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80901",
-    "pr_number": 80901,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80905": {
-    "commit": "c08ee9d51b31c5dcda54e291694feb1cd747b7eb",
-    "text": "Implement a new feature that allows applying kustomize patches to static pod manifests generated by kubeadm.",
-    "markdown": "Implement a new feature that allows applying kustomize patches to static pod manifests generated by kubeadm. ([#80905](https://github.com/kubernetes/kubernetes/pull/80905), [@fabriziopandini](https://github.com/fabriziopandini))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-cluster-lifecycle/kubeadm/20190722-Advanced-configurations-with-kubeadm-(Kustomize).md",
-        "type": "KEP"
-      }
-    ],
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80905",
-    "pr_number": 80905,
-    "areas": ["dependency", "kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80911": {
-    "commit": "77277d3abf9cd7289480c6c353da11ba7db9cc79",
-    "text": "Adds support for vSphere volumes on Windows",
-    "markdown": "Adds support for vSphere volumes on Windows ([#80911](https://github.com/kubernetes/kubernetes/pull/80911), [@gab-satchi](https://github.com/gab-satchi))\n\n  Courtesy of SIG Storage, and SIG Windows",
-    "author": "gab-satchi",
-    "author_url": "https://github.com/gab-satchi",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80911",
-    "pr_number": 80911,
-    "areas": ["provider/vmware"],
-    "kinds": ["feature"],
-    "sigs": ["storage", "windows"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80912": {
-    "commit": "6789f38199d20cf8fadf38d79b7484d8af0a6ebd",
-    "text": "updates fluentd-elasticsearch docker image to fluentd 1.6.3",
-    "markdown": "updates fluentd-elasticsearch docker image to fluentd 1.6.3 ([#80912](https://github.com/kubernetes/kubernetes/pull/80912), [@monotek](https://github.com/monotek))",
-    "author": "monotek",
-    "author_url": "https://github.com/monotek",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80912",
-    "pr_number": 80912,
-    "kinds": ["flake"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "80923": {
-    "commit": "5c16a784c694734060aeb7f4f650d0cf11216b9d",
-    "text": "Reduces GCE PD Node Attach Limits by 1 since the node boot disk is considered an attachable disk",
-    "markdown": "Reduces GCE PD Node Attach Limits by 1 since the node boot disk is considered an attachable disk ([#80923](https://github.com/kubernetes/kubernetes/pull/80923), [@davidz627](https://github.com/davidz627))",
-    "author": "davidz627",
-    "author_url": "https://github.com/davidz627",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80923",
-    "pr_number": 80923,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.16.0"
-  },
-  "80930": {
-    "commit": "1a53325550f6d5d3c48b9eecdd123fd84deee879",
-    "text": "NormalizeScore plugin set is removed from scheduler framework config API. Use ScorePlugin only.",
-    "markdown": "NormalizeScore plugin set is removed from scheduler framework config API. Use ScorePlugin only. ([#80930](https://github.com/kubernetes/kubernetes/pull/80930), [@liu-cong](https://github.com/liu-cong))",
-    "author": "liu-cong",
-    "author_url": "https://github.com/liu-cong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80930",
-    "pr_number": 80930,
-    "kinds": ["api-change"],
-    "sigs": ["scheduling"],
-    "release_version": "1.16.0"
-  },
-  "80937": {
-    "commit": "59f5b3ddc4ddcc7d400d3fd5624712e79929bffc",
-    "text": "kubeadm: print the stack trace of an error for klog level --v\u003e=5",
-    "markdown": "kubeadm: print the stack trace of an error for klog level --v\u003e=5 ([#80937](https://github.com/kubernetes/kubernetes/pull/80937), [@neolit123](https://github.com/neolit123))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80937",
-    "pr_number": 80937,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80942": {
-    "commit": "12a085f9176cfd7fa4514922f6481a3d1787b207",
-    "text": "Fix a bug in the IPVS proxier where virtual servers are not cleaned up even though the corresponding Service object was deleted.",
-    "markdown": "Fix a bug in the IPVS proxier where virtual servers are not cleaned up even though the corresponding Service object was deleted. ([#80942](https://github.com/kubernetes/kubernetes/pull/80942), [@gongguan](https://github.com/gongguan))",
-    "author": "gongguan",
-    "author_url": "https://github.com/gongguan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80942",
-    "pr_number": 80942,
-    "areas": ["ipvs"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.16.0"
-  },
-  "80945": {
-    "commit": "6049253aae49f87985a59ba45bedded94023b093",
-    "text": "Fixes validation of VolumeAttachment API objects created with inline volume sources.",
-    "markdown": "Fixes validation of VolumeAttachment API objects created with inline volume sources. ([#80945](https://github.com/kubernetes/kubernetes/pull/80945), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80945",
-    "pr_number": 80945,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.16.0"
-  },
-  "80951": {
-    "commit": "74c0cc27902548243d8a863c9e2c6345a8e6b548",
-    "text": "kubeadm: enable secure serving for the kube-scheduler",
-    "markdown": "kubeadm: enable secure serving for the kube-scheduler ([#80951](https://github.com/kubernetes/kubernetes/pull/80951), [@neolit123](https://github.com/neolit123))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80951",
-    "pr_number": 80951,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "80966": {
-    "commit": "da30406a9d993f437c5d971f33f44a739312a914",
-    "text": "kubeadm: fix \"certificate-authority\" files not being pre-loaded when using file discovery",
-    "markdown": "kubeadm: fix \"certificate-authority\" files not being pre-loaded when using file discovery ([#80966](https://github.com/kubernetes/kubernetes/pull/80966), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80966",
-    "pr_number": 80966,
-    "areas": ["kubeadm"],
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "80978": {
-    "commit": "ef8869466c803fbbe44a411a8a2e3e37c7f1d1d2",
-    "text": "API: the metadata.selfLink field is deprecated in individual and list objects. It will no longer be returned starting in v1.20, and the field will be removed entirely in v1.21.",
-    "markdown": "API: the metadata.selfLink field is deprecated in individual and list objects. It will no longer be returned starting in v1.20, and the field will be removed entirely in v1.21. ([#80978](https://github.com/kubernetes/kubernetes/pull/80978), [@wojtek-t](https://github.com/wojtek-t))",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80978",
-    "pr_number": 80978,
-    "areas": ["apiserver", "code-generation", "kubelet"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "apps", "node"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81015": {
-    "commit": "7a35f8c92dd5125924873fe4db8fd82b8889c99a",
-    "text": "Return error when the scoring plugin returns score out of range [0, 100].",
-    "markdown": "Return error when the scoring plugin returns score out of range [0, 100]. ([#81015](https://github.com/kubernetes/kubernetes/pull/81015), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-scheduling/20180409-scheduling-framework.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81015",
-    "pr_number": 81015,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81029": {
-    "commit": "62f1c4065b4c4b1615833245384e8de81bd530d9",
-    "text": "cloud-controller-manager binaries and docker images are no longer shipped with kubernetes releases.",
-    "markdown": "cloud-controller-manager binaries and docker images are no longer shipped with kubernetes releases. ([#81029](https://github.com/kubernetes/kubernetes/pull/81029), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81029",
-    "pr_number": 81029,
-    "kinds": ["cleanup"],
-    "sigs": ["cloud-provider", "release"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81043": {
-    "commit": "0f46a8a4c87a092e536b5af538dde60b0cf77736",
-    "text": "/test/e2e/framework: Adds a flag \"non-blocking-taints\" which allows tests to run in environments with tainted nodes. String value should be a comma-separated list.",
-    "markdown": "/test/e2e/framework: Adds a flag \"non-blocking-taints\" which allows tests to run in environments with tainted nodes. String value should be a comma-separated list. ([#81043](https://github.com/kubernetes/kubernetes/pull/81043), [@johnSchnake](https://github.com/johnSchnake))\n\n  Courtesy of SIG Testing",
-    "author": "johnSchnake",
-    "author_url": "https://github.com/johnSchnake",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81043",
-    "pr_number": 81043,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["feature"],
-    "sigs": ["testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81048": {
-    "commit": "7da9bdc4f64a5014a4b45cc34c8a86976db1ce11",
-    "text": "Adds Endpoint Slice Controller for managing new EndpointSlice resource, disabled by default.",
-    "markdown": "Adds Endpoint Slice Controller for managing new EndpointSlice resource, disabled by default. ([#81048](https://github.com/kubernetes/kubernetes/pull/81048), [@robscott](https://github.com/robscott))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG Network, and SIG Testing",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81048",
-    "pr_number": 81048,
-    "areas": ["apiserver", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "auth", "network", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81054": {
-    "commit": "8527c74e40f0b3b8cf5f5fe80e4fff607a147c1a",
-    "text": "Add a azure cloud configuration `LoadBalancerName` and `LoadBalancerResourceGroup` to allow the corresponding customizations of azure load balancer.",
-    "markdown": "Add a azure cloud configuration `LoadBalancerName` and `LoadBalancerResourceGroup` to allow the corresponding customizations of azure load balancer. ([#81054](https://github.com/kubernetes/kubernetes/pull/81054), [@nilo19](https://github.com/nilo19))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81054",
-    "pr_number": 81054,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81094": {
-    "commit": "9690201481415a548dff6866d836c2ad2a87d585",
-    "text": "Deprecate the `--cloud-provider-gce-lb-src-cidrs` flag in the kube-apiserver. This flag will be removed once the GCE Cloud Provider is removed from kube-apiserver.",
-    "markdown": "Deprecate the `--cloud-provider-gce-lb-src-cidrs` flag in the kube-apiserver. This flag will be removed once the GCE Cloud Provider is removed from kube-apiserver. ([#81094](https://github.com/kubernetes/kubernetes/pull/81094), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81094",
-    "pr_number": 81094,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "network"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81096": {
-    "commit": "3030a9d92750173ed72ea9cd37286bf6a41c2dcb",
-    "text": "Fix a bug that pretty printer marshals empty byte or uint8 slice as null",
-    "markdown": "Fix a bug that pretty printer marshals empty byte or uint8 slice as null ([#81096](https://github.com/kubernetes/kubernetes/pull/81096), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81096",
-    "pr_number": 81096,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81106": {
-    "commit": "18b6ff3d65a77b1b257af6e1128b322c7de0031b",
-    "text": "Update the GCE windows node image to include hot fixes since July.",
-    "markdown": "Update the GCE windows node image to include hot fixes since July. ([#81106](https://github.com/kubernetes/kubernetes/pull/81106), [@YangLu1031](https://github.com/YangLu1031))",
-    "author": "YangLu1031",
-    "author_url": "https://github.com/YangLu1031",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81106",
-    "pr_number": 81106,
-    "kinds": ["cleanup", "flake"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "81152": {
-    "commit": "f4e39afea082afac7bf6f6936c34e012b7cb3c25",
-    "text": "kube-apiserver: the `--basic-auth-file` flag and authentication mode is deprecated and will be removed in a future release. It is not recommended for production environments.",
-    "markdown": "kube-apiserver: the `--basic-auth-file` flag and authentication mode is deprecated and will be removed in a future release. It is not recommended for production environments. ([#81152](https://github.com/kubernetes/kubernetes/pull/81152), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81152",
-    "pr_number": 81152,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery", "auth"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81163": {
-    "commit": "e319abf2745b5c70f55a2b8b03bfac56a5ee088f",
-    "text": "Volumes specified in a pod but not used in it are no longer unnecessarily formatted, mounted and reported in `node.status.volumesInUse`.",
-    "markdown": "Volumes specified in a pod but not used in it are no longer unnecessarily formatted, mounted and reported in `node.status.volumesInUse`. ([#81163](https://github.com/kubernetes/kubernetes/pull/81163), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81163",
-    "pr_number": 81163,
-    "areas": ["e2e-test-framework", "kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["node", "storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81174": {
-    "commit": "c0997e8dc1dfe5d804917e1d0c7d97d4c6fdde9e",
-    "text": "Fix kubelet NodeLease potential performance issues. Kubelet now will try to update lease using cached one instead of get from API Server every time.",
-    "markdown": "Fix kubelet NodeLease potential performance issues. Kubelet now will try to update lease using cached one instead of get from API Server every time. ([#81174](https://github.com/kubernetes/kubernetes/pull/81174), [@answer1991](https://github.com/answer1991))",
-    "author": "answer1991",
-    "author_url": "https://github.com/answer1991",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81174",
-    "pr_number": 81174,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.16.0"
-  },
-  "81208": {
-    "commit": "a4c5a578004b71e41fc05c167ea96b6016046ce6",
-    "text": "switch to VM Update call in attach/detach disk operation, original CreateOrUpdate call may lead to orphaned VMs or blocked resources",
-    "markdown": "switch to VM Update call in attach/detach disk operation, original CreateOrUpdate call may lead to orphaned VMs or blocked resources ([#81208](https://github.com/kubernetes/kubernetes/pull/81208), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81208",
-    "pr_number": 81208,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.0"
-  },
-  "81210": {
-    "commit": "406904269d5a757bedd57be4d49b83d5f57a5847",
-    "text": "added an new Prometheus counter metric \"sync_proxy_rules_iptables_restore_failures_total\" for kube-proxy iptables-restore failures (both ipvs and iptables modes)",
-    "markdown": "added an new Prometheus counter metric \"sync_proxy_rules_iptables_restore_failures_total\" for kube-proxy iptables-restore failures (both ipvs and iptables modes) ([#81210](https://github.com/kubernetes/kubernetes/pull/81210), [@figo](https://github.com/figo))\n\n  Courtesy of SIG Network",
-    "author": "figo",
-    "author_url": "https://github.com/figo",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81210",
-    "pr_number": 81210,
-    "areas": ["ipvs"],
-    "kinds": ["feature"],
-    "sigs": ["network"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81212": {
-    "commit": "2c455e0ac0aceeee74281e0f3fb199109921bb49",
-    "text": "Improves validation errors for custom resources",
-    "markdown": "Improves validation errors for custom resources ([#81212](https://github.com/kubernetes/kubernetes/pull/81212), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81212",
-    "pr_number": 81212,
-    "areas": ["custom-resources", "dependency"],
-    "kinds": ["api-change", "cleanup", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81213": {
-    "commit": "a12b64802959b4d826137e7eb1512f0058109bd3",
-    "text": "Add a service annotation `service.beta.kubernetes.io/azure-pip-name` to specify the public IP name for Azure load balancer.",
-    "markdown": "Add a service annotation `service.beta.kubernetes.io/azure-pip-name` to specify the public IP name for Azure load balancer. ([#81213](https://github.com/kubernetes/kubernetes/pull/81213), [@nilo19](https://github.com/nilo19))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81213",
-    "pr_number": 81213,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81215": {
-    "commit": "d23b2c7d83fc7b888a7d6dfd4ba940e8d7c06cb6",
-    "text": "remove iSCSI volume storage cleartext secrets in logs",
-    "markdown": "remove iSCSI volume storage cleartext secrets in logs ([#81215](https://github.com/kubernetes/kubernetes/pull/81215), [@zouyee](https://github.com/zouyee))",
-    "author": "zouyee",
-    "author_url": "https://github.com/zouyee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81215",
-    "pr_number": 81215,
-    "kinds": ["bug", "flake"],
-    "sigs": ["storage"],
-    "release_version": "1.16.0"
-  },
-  "81217": {
-    "commit": "ca6290538b91ebd6fa65408a5d4030efa8c15918",
-    "text": "kubeadm: the permissions of generated CSR files are changed from 0644 to 0600",
-    "markdown": "kubeadm: the permissions of generated CSR files are changed from 0644 to 0600 ([#81217](https://github.com/kubernetes/kubernetes/pull/81217), [@SataQiu](https://github.com/SataQiu))",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81217",
-    "pr_number": 81217,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "81219": {
-    "commit": "06fb339e31b127bf7d506a656b02188f303f7ddd",
-    "text": "hyperkube will drop support for cloud-controller-manager in a future release",
-    "markdown": "hyperkube will drop support for cloud-controller-manager in a future release ([#81219](https://github.com/kubernetes/kubernetes/pull/81219), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81219",
-    "pr_number": 81219,
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cloud-provider", "release"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81244": {
-    "commit": "3f0a486cbf43abe113923ed6947b8d338574ad43",
-    "text": "Resolves a transient 404 response to custom resource requests during server startup",
-    "markdown": "Resolves a transient 404 response to custom resource requests during server startup ([#81244](https://github.com/kubernetes/kubernetes/pull/81244), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81244",
-    "pr_number": 81244,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.0"
-  },
-  "81266": {
-    "commit": "1719ce7883db57d24d70605606f189469fd50b60",
-    "text": "fix: detach azure disk issue using dangling error",
-    "markdown": "fix: detach azure disk issue using dangling error ([#81266](https://github.com/kubernetes/kubernetes/pull/81266), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81266",
-    "pr_number": 81266,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81279": {
-    "commit": "54b28ab60c66746f314fe2ed109c46893f37a71e",
-    "text": "Fix Azure client requests stuck issues on http.StatusTooManyRequests (HTTP Code 429).",
-    "markdown": "Fix Azure client requests stuck issues on http.StatusTooManyRequests (HTTP Code 429). ([#81279](https://github.com/kubernetes/kubernetes/pull/81279), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81279",
-    "pr_number": 81279,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.0"
-  },
-  "81282": {
-    "commit": "ae457448d6963ffbb6427afc8c87598f89fadf71",
-    "text": "Fix conflicted cache when the requests are canceled by other Azure operations.",
-    "markdown": "Fix conflicted cache when the requests are canceled by other Azure operations. ([#81282](https://github.com/kubernetes/kubernetes/pull/81282), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81282",
-    "pr_number": 81282,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.0"
-  },
-  "81300": {
-    "commit": "9614a85a2b6b3122b9ca68c4992801648248dcd3",
-    "text": "Install and start logging agent based on kube env ENABLE_NODE_LOGGING.",
-    "markdown": "Install and start logging agent based on kube env ENABLE_NODE_LOGGING. ([#81300](https://github.com/kubernetes/kubernetes/pull/81300), [@liyanhui1228](https://github.com/liyanhui1228))",
-    "author": "liyanhui1228",
-    "author_url": "https://github.com/liyanhui1228",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81300",
-    "pr_number": 81300,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81306": {
-    "commit": "de3aed802338437f189313dcd6b17ddd9bb15e95",
-    "text": "Scheduler should terminate when it looses leader lock.",
-    "markdown": "Scheduler should terminate when it looses leader lock. ([#81306](https://github.com/kubernetes/kubernetes/pull/81306), [@ravisantoshgudimetla](https://github.com/ravisantoshgudimetla))",
-    "author": "ravisantoshgudimetla",
-    "author_url": "https://github.com/ravisantoshgudimetla",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81306",
-    "pr_number": 81306,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.16.0"
-  },
-  "81330": {
-    "commit": "4441f1d9c3e94d9a3d93b4f184a591cab02a5245",
-    "text": "Fixes CVE-2019-11250: client-go header logging (at verbosity levels \u003e= 7) now masks `Authorization` header contents",
-    "markdown": "Fixes CVE-2019-11250: client-go header logging (at verbosity levels \u003e= 7) now masks `Authorization` header contents ([#81330](https://github.com/kubernetes/kubernetes/pull/81330), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81330",
-    "pr_number": 81330,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81337": {
-    "commit": "2974adff27619bb27a27a78bd61de772a2d65bef",
-    "text": "Add instruction to setup \"Application Default Credentials\" to run GCE Windows e2e tests locally.",
-    "markdown": "Add instruction to setup \"Application Default Credentials\" to run GCE Windows e2e tests locally. ([#81337](https://github.com/kubernetes/kubernetes/pull/81337), [@YangLu1031](https://github.com/YangLu1031))",
-    "author": "YangLu1031",
-    "author_url": "https://github.com/YangLu1031",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81337",
-    "pr_number": 81337,
-    "kinds": ["documentation"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "81342": {
-    "commit": "f81ab5a9fca3212386fd1eb0fb40950f40f55a33",
-    "text": "kubectl could scale custom resource again",
-    "markdown": "kubectl could scale custom resource again ([#81342](https://github.com/kubernetes/kubernetes/pull/81342), [@knight42](https://github.com/knight42))\n\n  Courtesy of SIG CLI, and SIG Testing",
-    "author": "knight42",
-    "author_url": "https://github.com/knight42",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81342",
-    "pr_number": 81342,
-    "areas": ["e2e-test-framework", "kubectl", "test"],
-    "kinds": ["feature"],
-    "sigs": ["cli", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81377": {
-    "commit": "3645041af5ec0a4f55a3e9eaa4cd3aec402cca96",
-    "text": "kubelet now exports an \"kubelet_evictions\" metric that counts the number of pod evictions carried out by the kubelet to reclaim resources",
-    "markdown": "kubelet now exports an \"kubelet_evictions\" metric that counts the number of pod evictions carried out by the kubelet to reclaim resources ([#81377](https://github.com/kubernetes/kubernetes/pull/81377), [@sjenning](https://github.com/sjenning))\n\n  Courtesy of SIG Instrumentation, and SIG Node",
-    "author": "sjenning",
-    "author_url": "https://github.com/sjenning",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81377",
-    "pr_number": 81377,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["instrumentation", "node"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81385": {
-    "commit": "9e60bedebed6be97a91dda82451e8f82d66c34d4",
-    "text": "kubeadm: use etcd's /health endpoint for a HTTP liveness probe on localhost instead of having a custom health check using etcdctl",
-    "markdown": "kubeadm: use etcd's /health endpoint for a HTTP liveness probe on localhost instead of having a custom health check using etcdctl ([#81385](https://github.com/kubernetes/kubernetes/pull/81385), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81385",
-    "pr_number": 81385,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "81390": {
-    "commit": "3526a6162e7165966c3adb811ef82d20fabe17c8",
-    "text": "Update to use go 1.12.8",
-    "markdown": "Update to use go 1.12.8 ([#81390](https://github.com/kubernetes/kubernetes/pull/81390), [@cblecker](https://github.com/cblecker))",
-    "author": "cblecker",
-    "author_url": "https://github.com/cblecker",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81390",
-    "pr_number": 81390,
-    "areas": ["cloudprovider", "test"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "release", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81394": {
-    "commit": "1f6cb3cb9def97320a5412dcbea1661edd95c29e",
-    "text": "Update golang/x/net dependency to bring in fixes for CVE-2019-9512, CVE-2019-9514",
-    "markdown": "Update golang/x/net dependency to bring in fixes for CVE-2019-9512, CVE-2019-9514 ([#81394](https://github.com/kubernetes/kubernetes/pull/81394), [@cblecker](https://github.com/cblecker))",
-    "author": "cblecker",
-    "author_url": "https://github.com/cblecker",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81394",
-    "pr_number": 81394,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "node", "release"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81397": {
-    "commit": "08b67378d3557734caab683e856c8e09ce1d7876",
-    "text": "Support Kubelet plugin watcher on Windows nodes.",
-    "markdown": "Support Kubelet plugin watcher on Windows nodes. ([#81397](https://github.com/kubernetes/kubernetes/pull/81397), [@ddebroy](https://github.com/ddebroy))\n\n  Courtesy of SIG Node, and SIG Windows",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-windows/20190714-windows-csi-support.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "ddebroy",
-    "author_url": "https://github.com/ddebroy",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81397",
-    "pr_number": 81397,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node", "windows"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81399": {
-    "commit": "34605737b9a96ee4b94ebf8f970501e238086370",
-    "text": "The `rejected` label in `apiserver_admission_webhook_admission_duration_seconds` metrices now properly indicates if a request was rejected. Add a new counter metrics `apiserver_admission_webhook_rejection_count` with details about the causing for a webhook rejection.",
-    "markdown": "The `rejected` label in `apiserver_admission_webhook_admission_duration_seconds` metrices now properly indicates if a request was rejected. Add a new counter metrics `apiserver_admission_webhook_rejection_count` with details about the causing for a webhook rejection. ([#81399](https://github.com/kubernetes/kubernetes/pull/81399), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81399",
-    "pr_number": 81399,
-    "areas": ["apiserver"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81411": {
-    "commit": "11eaf430e1ed0b4bc18dbc18321c99a94ee96911",
-    "text": "Fix VMSS LoadBalancer backend pools so that the network won't be broken when instances are upgraded to latest model",
-    "markdown": "Fix VMSS LoadBalancer backend pools so that the network won't be broken when instances are upgraded to latest model ([#81411](https://github.com/kubernetes/kubernetes/pull/81411), [@nilo19](https://github.com/nilo19))",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81411",
-    "pr_number": 81411,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.0"
-  },
-  "81413": {
-    "commit": "443002fcb7bdd5261e383b6da358a16e43183415",
-    "text": "The nbf (not before) claim, if present in ID token, is now enforced.",
-    "markdown": "The nbf (not before) claim, if present in ID token, is now enforced. ([#81413](https://github.com/kubernetes/kubernetes/pull/81413), [@anderseknert](https://github.com/anderseknert))\n\n  Courtesy of SIG API Machinery, SIG Auth, and SIG Cloud Provider",
-    "author": "anderseknert",
-    "author_url": "https://github.com/anderseknert",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81413",
-    "pr_number": 81413,
-    "areas": ["apiserver", "cloudprovider", "dependency"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "auth", "cloud-provider"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81429": {
-    "commit": "f105fef3d5ca0c78b056bf826ee546bf1107cf8b",
-    "text": "Resolves an issue that prevented block volumes from being resized.",
-    "markdown": "Resolves an issue that prevented block volumes from being resized. ([#81429](https://github.com/kubernetes/kubernetes/pull/81429), [@huffmanca](https://github.com/huffmanca))",
-    "author": "huffmanca",
-    "author_url": "https://github.com/huffmanca",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81429",
-    "pr_number": 81429,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "storage"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81430": {
-    "commit": "4495d09282265964ae217ab9976c4c4ff5945b7b",
-    "text": "Adds EndpointSlice integration to kube-proxy, can be enabled with EndpointSlice feature gate.",
-    "markdown": "Adds EndpointSlice integration to kube-proxy, can be enabled with EndpointSlice feature gate. ([#81430](https://github.com/kubernetes/kubernetes/pull/81430), [@robscott](https://github.com/robscott))\n\n  Courtesy of SIG API Machinery, SIG Network, and SIG Testing",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81430",
-    "pr_number": 81430,
-    "areas": ["apiserver", "ipvs", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "network", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81436": {
-    "commit": "21cdcd577609c039043903edfd4995077fc6b27d",
-    "text": "Fix a bug in CRD openapi controller that user-defined CRD can overwrite OpenAPI definition/path for the CRD API.",
-    "markdown": "Fix a bug in CRD openapi controller that user-defined CRD can overwrite OpenAPI definition/path for the CRD API. ([#81436](https://github.com/kubernetes/kubernetes/pull/81436), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81436",
-    "pr_number": 81436,
-    "areas": ["dependency", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81467": {
-    "commit": "dccd28269a6596bf2f0041c14dd3233be8cfa571",
-    "text": "Move CSI volume expansion to beta.",
-    "markdown": "Move CSI volume expansion to beta. ([#81467](https://github.com/kubernetes/kubernetes/pull/81467), [@bertinatto](https://github.com/bertinatto))\n\n  Courtesy of SIG Testing",
-    "documentation": [
-      {
-        "url": "https://github.com/kubernetes/enhancements/issues/556",
-        "type": "KEP"
-      }
-    ],
-    "author": "bertinatto",
-    "author_url": "https://github.com/bertinatto",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81467",
-    "pr_number": 81467,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81468": {
-    "commit": "e102d575f4625a1d5bd1178391d84c5559df1dbe",
-    "text": "kubectl: the --all-namespaces flag is now honored by `kubectl wait`",
-    "markdown": "kubectl: the --all-namespaces flag is now honored by `kubectl wait` ([#81468](https://github.com/kubernetes/kubernetes/pull/81468), [@ashutoshgngwr](https://github.com/ashutoshgngwr))",
-    "author": "ashutoshgngwr",
-    "author_url": "https://github.com/ashutoshgngwr",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81468",
-    "pr_number": 81468,
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.16.0"
-  },
-  "81476": {
-    "commit": "770303eaee8cc50374975889e1fd1d2db869a8f8",
-    "text": "Conversion webhooks can now indicate they support receiving and responding with `ConversionReview` API objects in the `apiextensions.k8s.io/v1` version by including `v1` in the `conversionReviewVersions` list in their CustomResourceDefinition. Conversion webhooks must respond with a ConversionReview object in the same apiVersion they receive. `apiextensions.k8s.io/v1` `ConversionReview` responses must specify a `response.uid` that matches the `request.uid` of the object they were sent.",
-    "markdown": "Conversion webhooks can now indicate they support receiving and responding with `ConversionReview` API objects in the `apiextensions.k8s.io/v1` version by including `v1` in the `conversionReviewVersions` list in their CustomResourceDefinition. Conversion webhooks must respond with a ConversionReview object in the same apiVersion they receive. `apiextensions.k8s.io/v1` `ConversionReview` responses must specify a `response.uid` that matches the `request.uid` of the object they were sent. ([#81476](https://github.com/kubernetes/kubernetes/pull/81476), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery, and SIG Testing",
-    "documentation": [
-      {
-        "description": "KEP",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190425-crd-conversion-webhook.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81476",
-    "pr_number": 81476,
-    "areas": ["apiserver", "custom-resources", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81477": {
-    "commit": "bd8a8db515381b3058e211822a00040662109911",
-    "text": "Fix in kube-proxy for SCTP nodeport service which only works for node's InternalIP, but doesn't work for other IPs present in the node when ipvs is enabled.",
-    "markdown": "Fix in kube-proxy for SCTP nodeport service which only works for node's InternalIP, but doesn't work for other IPs present in the node when ipvs is enabled. ([#81477](https://github.com/kubernetes/kubernetes/pull/81477), [@paulsubrata55](https://github.com/paulsubrata55))",
-    "author": "paulsubrata55",
-    "author_url": "https://github.com/paulsubrata55",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81477",
-    "pr_number": 81477,
-    "areas": ["ipvs"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.16.0"
-  },
-  "81489": {
-    "commit": "e283836fb2ced2c1d918eef43c837f27c9381fc9",
-    "text": "Update to use go 1.12.9",
-    "markdown": "Update to use go 1.12.9 ([#81489](https://github.com/kubernetes/kubernetes/pull/81489), [@BenTheElder](https://github.com/BenTheElder))",
-    "author": "BenTheElder",
-    "author_url": "https://github.com/BenTheElder",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81489",
-    "pr_number": 81489,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["release", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81494": {
-    "commit": "17a1859370785fe94150b7489215c1a2aea24dfa",
-    "text": "kubeadm reset: unmount directories under \"/var/lib/kubelet\" for linux only",
-    "markdown": "kubeadm reset: unmount directories under \"/var/lib/kubelet\" for linux only ([#81494](https://github.com/kubernetes/kubernetes/pull/81494), [@Klaven](https://github.com/Klaven))",
-    "author": "Klaven",
-    "author_url": "https://github.com/Klaven",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81494",
-    "pr_number": 81494,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "81500": {
-    "commit": "667ea63ec2f96525f45848f0b6a60b512b3911a7",
-    "text": "Kubelet could be run with no Azure identity without subscriptionId configured now.\nA sample cloud provider configure is: '{\"vmType\": \"vmss\", \"useInstanceMetadata\": true}'.",
-    "markdown": "Kubelet could be run with no Azure identity without subscriptionId configured now.\n  A sample cloud provider configure is: '{\"vmType\": \"vmss\", \"useInstanceMetadata\": true}'. ([#81500](https://github.com/kubernetes/kubernetes/pull/81500), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81500",
-    "pr_number": 81500,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.0"
-  },
-  "81509": {
-    "commit": "30f2545364f3225b62651e26b872e5abdf21ab0b",
-    "text": "Added metrics 'authentication_attempts' that can be used to understand the attempts of authentication.",
-    "markdown": "Added metrics 'authentication_attempts' that can be used to understand the attempts of authentication. ([#81509](https://github.com/kubernetes/kubernetes/pull/81509), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG API Machinery, SIG Auth, and SIG Instrumentation",
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81509",
-    "pr_number": 81509,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "auth", "instrumentation"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81515": {
-    "commit": "c7e997b5271bdf7d27fdd7957361bc12133b35c5",
-    "text": "openapi now advertises correctly supported patch types for custom resources",
-    "markdown": "openapi now advertises correctly supported patch types for custom resources ([#81515](https://github.com/kubernetes/kubernetes/pull/81515), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81515",
-    "pr_number": 81515,
-    "areas": ["custom-resources"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.0"
-  },
-  "81531": {
-    "commit": "f24aa47ceab19d174117d80eec657875c06aeb0c",
-    "text": "Kube-apiserver metrics are now marked as with the ALPHA stability level.",
-    "markdown": "Kube-apiserver metrics are now marked as with the ALPHA stability level. ([#81531](https://github.com/kubernetes/kubernetes/pull/81531), [@logicalhan](https://github.com/logicalhan))\n\n  Courtesy of SIG API Machinery, SIG Auth, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Instrumentation, and SIG Release",
-    "documentation": [
-      {
-        "description": "[KEP Metrics Stability Migration]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190605-metrics-stability-migration.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81531",
-    "pr_number": 81531,
-    "areas": ["apiserver", "cloudprovider", "dependency", "release-eng"],
-    "kinds": ["feature"],
-    "sigs": [
-      "api-machinery",
-      "auth",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "release"
-    ],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81534": {
-    "commit": "ca5babc1da4f02998ceab90e3d2de57614069c45",
-    "text": "Kubelet metrics for /metrics and /metrics/probes are now marked as with the ALPHA stability level.",
-    "markdown": "Kubelet metrics for /metrics and /metrics/probes are now marked as with the ALPHA stability level. ([#81534](https://github.com/kubernetes/kubernetes/pull/81534), [@logicalhan](https://github.com/logicalhan))\n\n  Courtesy of SIG API Machinery, SIG Auth, SIG Cluster Lifecycle, SIG Instrumentation, SIG Network, SIG Node, SIG Release, and SIG Storage",
-    "documentation": [
-      {
-        "description": "[KEP Metrics Stability Migration]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190605-metrics-stability-migration.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81534",
-    "pr_number": 81534,
-    "areas": ["dependency", "kubelet", "release-eng"],
-    "kinds": ["feature"],
-    "sigs": [
-      "api-machinery",
-      "auth",
-      "cluster-lifecycle",
-      "instrumentation",
-      "network",
-      "node",
-      "release",
-      "storage"
-    ],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81549": {
-    "commit": "a7e9704c0e8ab6584e2177d59ddd2fc733af3b63",
-    "text": "Added support to specify a global-access annotation for gce ILB.",
-    "markdown": "Added support to specify a global-access annotation for gce ILB. ([#81549](https://github.com/kubernetes/kubernetes/pull/81549), [@prameshj](https://github.com/prameshj))",
-    "author": "prameshj",
-    "author_url": "https://github.com/prameshj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81549",
-    "pr_number": 81549,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "node"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81573": {
-    "commit": "7da563f0f80d45b21e9008691c142c7f75350cc7",
-    "text": "Add `container_state` label to `running_container_count` kubelet metrics, to get count of containers based on their state(running/exited/created/unknown)",
-    "markdown": "Add `container_state` label to `running_container_count` kubelet metrics, to get count of containers based on their state(running/exited/created/unknown) ([#81573](https://github.com/kubernetes/kubernetes/pull/81573), [@irajdeep](https://github.com/irajdeep))",
-    "author": "irajdeep",
-    "author_url": "https://github.com/irajdeep",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81573",
-    "pr_number": 81573,
-    "areas": ["kubelet"],
-    "kinds": ["design"],
-    "sigs": ["instrumentation", "node"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81576": {
-    "commit": "a927ed454cee3eea5f5acf9ff837075fb372dd0c",
-    "text": "Scheduler metrics are now marked as with the ALPHA stability level.",
-    "markdown": "Scheduler metrics are now marked as with the ALPHA stability level. ([#81576](https://github.com/kubernetes/kubernetes/pull/81576), [@logicalhan](https://github.com/logicalhan))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Instrumentation, SIG Release, and SIG Scheduling",
-    "documentation": [
-      {
-        "description": "[KEP Metrics Stability Migration]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190605-metrics-stability-migration.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81576",
-    "pr_number": 81576,
-    "areas": ["apiserver", "cloudprovider", "dependency", "release-eng"],
-    "kinds": ["feature"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "auth",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "release",
-      "scheduling"
-    ],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81623": {
-    "commit": "ca3e86e662fda8310c071e4660ef5703997f1409",
-    "text": "kubeadm: support any Linux kernel version newer than 3.10",
-    "markdown": "kubeadm: support any Linux kernel version newer than 3.10 ([#81623](https://github.com/kubernetes/kubernetes/pull/81623), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81623",
-    "pr_number": 81623,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "81624": {
-    "commit": "d6bc4eb853327437b25f34119a78015ca9565809",
-    "text": "Kube-controller-manager and cloud-controller-manager metrics are now marked as with the ALPHA stability level.",
-    "markdown": "Kube-controller-manager and cloud-controller-manager metrics are now marked as with the ALPHA stability level. ([#81624](https://github.com/kubernetes/kubernetes/pull/81624), [@logicalhan](https://github.com/logicalhan))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Instrumentation, SIG Release, and SIG Storage",
-    "documentation": [
-      {
-        "description": "[KEP Metrics Stability Migration]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190605-metrics-stability-migration.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81624",
-    "pr_number": 81624,
-    "areas": ["apiserver", "cloudprovider", "dependency", "release-eng"],
-    "kinds": ["feature"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "auth",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "release",
-      "storage"
-    ],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81625": {
-    "commit": "5e649417c6e8b9baf836f09a581b342dfc424430",
-    "text": "Windows nodes on GCE now run with Windows Defender enabled.",
-    "markdown": "Windows nodes on GCE now run with Windows Defender enabled. ([#81625](https://github.com/kubernetes/kubernetes/pull/81625), [@pjh](https://github.com/pjh))",
-    "author": "pjh",
-    "author_url": "https://github.com/pjh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81625",
-    "pr_number": 81625,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81626": {
-    "commit": "7d241f0ec1b284643f2c0426f7fe62863328a2e9",
-    "text": "Kube-proxy metrics are now marked as with the ALPHA stability level.",
-    "markdown": "Kube-proxy metrics are now marked as with the ALPHA stability level. ([#81626](https://github.com/kubernetes/kubernetes/pull/81626), [@logicalhan](https://github.com/logicalhan))\n\n  Courtesy of SIG API Machinery, SIG Cluster Lifecycle, SIG Instrumentation, SIG Network, SIG Node, and SIG Release",
-    "documentation": [
-      {
-        "description": "[KEP Metrics Stability Migration]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190605-metrics-stability-migration.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81626",
-    "pr_number": 81626,
-    "areas": ["dependency", "kubelet", "release-eng"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "instrumentation", "network", "node", "release"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81634": {
-    "commit": "22a8bcf30af52d87a0289bb705ff5c41aaf6f72a",
-    "text": "Fixes a bug that when there is a \"connection refused\" error, the reflector's ListAndWatch func will return directly but what expected is that sleep 1 second and rewatch since the specified resourceVersion.",
-    "markdown": "Fixes a bug that when there is a \"connection refused\" error, the reflector's ListAndWatch func will return directly but what expected is that sleep 1 second and rewatch since the specified resourceVersion. ([#81634](https://github.com/kubernetes/kubernetes/pull/81634), [@likakuli](https://github.com/likakuli))",
-    "author": "likakuli",
-    "author_url": "https://github.com/likakuli",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81634",
-    "pr_number": 81634,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.0"
-  },
-  "81668": {
-    "commit": "1d016cc1d3ee2f95c5ea13e5666bb78d66c7c5e8",
-    "text": "Deprecate the default service IP CIDR. The previous default was `10.0.0.0/24` which will be removed in 6 months/2 releases. Cluster admins must specify their own desired value, by using `--service-cluster-ip-range` on kube-apiserver.",
-    "markdown": "Deprecate the default service IP CIDR. The previous default was `10.0.0.0/24` which will be removed in 6 months/2 releases. Cluster admins must specify their own desired value, by using `--service-cluster-ip-range` on kube-apiserver. ([#81668](https://github.com/kubernetes/kubernetes/pull/81668), [@darshanime](https://github.com/darshanime))\n\n  Courtesy of SIG API Machinery, and SIG Network",
-    "author": "darshanime",
-    "author_url": "https://github.com/darshanime",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81668",
-    "pr_number": 81668,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "network"],
-    "action_required": true,
-    "release_version": "1.16.0"
-  },
-  "81682": {
-    "commit": "36e0ea6bfaa9da19eff5e65f48b47f8f42efdcd6",
-    "text": "The RemainingItemCount feature is now beta.\n\nremainingItemCount is the number of subsequent items in the list which are not included in this list response. If the list request contained label or field selectors, then the number of remaining items is unknown and the field will be left unset and omitted during serialization. If the list is complete (either because it is not chunking or because this is the last chunk), then there are no more remaining items and this field will be left unset and omitted during serialization. Servers older than v1.15 do not set this field. The intended use of the remainingItemCount is *estimating- the size of a collection. Clients should not rely on the remainingItemCount to be set or to be exact.",
-    "markdown": "The RemainingItemCount feature is now beta.\n  \n  remainingItemCount is the number of subsequent items in the list which are not included in this list response. If the list request contained label or field selectors, then the number of remaining items is unknown and the field will be left unset and omitted during serialization. If the list is complete (either because it is not chunking or because this is the last chunk), then there are no more remaining items and this field will be left unset and omitted during serialization. Servers older than v1.15 do not set this field. The intended use of the remainingItemCount is *estimating- the size of a collection. Clients should not rely on the remainingItemCount to be set or to be exact. ([#81682](https://github.com/kubernetes/kubernetes/pull/81682), [@caesarxuchao](https://github.com/caesarxuchao))",
-    "author": "caesarxuchao",
-    "author_url": "https://github.com/caesarxuchao",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81682",
-    "pr_number": 81682,
-    "areas": ["apiserver", "code-generation", "kubectl"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "cli"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81691": {
-    "commit": "ab13cf1055098ce0ede00e9228e34ced098a9ab4",
-    "text": "Finalizer Protection for Service LoadBalancers is now in Beta (enabled by default). This feature ensures the Service resource is not fully deleted until the correlating load balancer resources are deleted.",
-    "markdown": "Finalizer Protection for Service LoadBalancers is now in Beta (enabled by default). This feature ensures the Service resource is not fully deleted until the correlating load balancer resources are deleted. ([#81691](https://github.com/kubernetes/kubernetes/pull/81691), [@MrHohn](https://github.com/MrHohn))\n\n  Courtesy of SIG Apps, SIG Network, and SIG Testing",
-    "author": "MrHohn",
-    "author_url": "https://github.com/MrHohn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81691",
-    "pr_number": 81691,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["feature"],
-    "sigs": ["apps", "network", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81720": {
-    "commit": "34450e9a2a5abd2a58980a813c4445c9175c0eab",
-    "text": "fix azure disk naming matching issue due to case sensitive comparison",
-    "markdown": "fix azure disk naming matching issue due to case sensitive comparison ([#81720](https://github.com/kubernetes/kubernetes/pull/81720), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81720",
-    "pr_number": 81720,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.0"
-  },
-  "81732": {
-    "commit": "0eb1bfc9b57e4915f205048757c98d5319e615e1",
-    "text": "Fix an issue with toleration merging \u0026 whitelist checking in the PodTolerationRestriction admission controller.",
-    "markdown": "Fix an issue with toleration merging \u0026 whitelist checking in the PodTolerationRestriction admission controller. ([#81732](https://github.com/kubernetes/kubernetes/pull/81732), [@tallclair](https://github.com/tallclair))",
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81732",
-    "pr_number": 81732,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.16.0"
-  },
-  "81747": {
-    "commit": "0e1bad37641c8ad790f470ed56c814cdb4e29dd2",
-    "text": "Fix `kubectl logs -f` for windows server containers.",
-    "markdown": "Fix `kubectl logs -f` for windows server containers. ([#81747](https://github.com/kubernetes/kubernetes/pull/81747), [@Random-Liu](https://github.com/Random-Liu))",
-    "author": "Random-Liu",
-    "author_url": "https://github.com/Random-Liu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81747",
-    "pr_number": 81747,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "windows"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81786": {
-    "commit": "bfd8610ddafb2fcb2cb330187c8248777ea0b022",
-    "text": "Log when kube-apiserver regenerates the OpenAPI spec and why. OpenAPI spec generation is a very CPU-heavy process that is sensitive to continuous updates of CRDs and APIServices.\n\nAdded metrics aggregator_openapi_v2_regeneration_count, aggregator_openapi_v2_regeneration_gauge and apiextension_openapi_v2_regeneration_count metrics counting the triggering APIService and CRDs and the reason (add, update, delete).",
-    "markdown": "Log when kube-apiserver regenerates the OpenAPI spec and why. OpenAPI spec generation is a very CPU-heavy process that is sensitive to continuous updates of CRDs and APIServices.\n  \n  Added metrics aggregator_openapi_v2_regeneration_count, aggregator_openapi_v2_regeneration_gauge and apiextension_openapi_v2_regeneration_count metrics counting the triggering APIService and CRDs and the reason (add, update, delete). ([#81786](https://github.com/kubernetes/kubernetes/pull/81786), [@sttts](https://github.com/sttts))\n\n  Courtesy of SIG API Machinery",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81786",
-    "pr_number": 81786,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81791": {
-    "commit": "90cf1891526ca8a131f59ac61e12f2855451f759",
-    "text": "kubeadm no longer performs IPVS checks as part of its preflight checks",
-    "markdown": "kubeadm no longer performs IPVS checks as part of its preflight checks ([#81791](https://github.com/kubernetes/kubernetes/pull/81791), [@yastij](https://github.com/yastij))",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81791",
-    "pr_number": 81791,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "81792": {
-    "commit": "b36265524286bd454aab36dbce944e4599dd2bf7",
-    "text": "Promotes VolumePVCDataSource (Cloning) feature to beta for 1.16 release",
-    "markdown": "Promotes VolumePVCDataSource (Cloning) feature to beta for 1.16 release ([#81792](https://github.com/kubernetes/kubernetes/pull/81792), [@j-griffith](https://github.com/j-griffith))\n\n  Courtesy of SIG Storage, and SIG Testing",
-    "author": "j-griffith",
-    "author_url": "https://github.com/j-griffith",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81792",
-    "pr_number": 81792,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["storage", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81795": {
-    "commit": "0be2f49268cab3154597794c29238f2c362e0d11",
-    "text": "Adds Endpoint Slice support for kubectl when discovery API group is enabled.",
-    "markdown": "Adds Endpoint Slice support for kubectl when discovery API group is enabled. ([#81795](https://github.com/kubernetes/kubernetes/pull/81795), [@robscott](https://github.com/robscott))\n\n  Courtesy of SIG API Machinery, SIG CLI, and SIG Testing",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81795",
-    "pr_number": 81795,
-    "areas": ["apiserver", "kubectl", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "cli", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81800": {
-    "commit": "3ede02028fe99222c6c905aeabc9d2b385c04120",
-    "text": "Use PostFilter instead of Postfilter in the scheduling framework\nUse PreFilter instead of Prefilter in the scheduling framework\nUse PreBind instead of Prebind in the scheduling framework",
-    "markdown": "Use PostFilter instead of Postfilter in the scheduling framework\n  Use PreFilter instead of Prefilter in the scheduling framework\n  Use PreBind instead of Prebind in the scheduling framework ([#81800](https://github.com/kubernetes/kubernetes/pull/81800), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81800",
-    "pr_number": 81800,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81872": {
-    "commit": "45b0b784461f8146a539aacaa6a4006fb4a83013",
-    "text": "The CustomResourceDefaulting feature is promoted to beta and enabled by default. Defaults may be specified in structural schemas via the `apiextensions.k8s.io/v1` API. See https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/\u0026#35;specifying-a-structural-schema for details.",
-    "markdown": "The CustomResourceDefaulting feature is promoted to beta and enabled by default. Defaults may be specified in structural schemas via the `apiextensions.k8s.io/v1` API. See https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/\u0026#35;specifying-a-structural-schema for details. ([#81872](https://github.com/kubernetes/kubernetes/pull/81872), [@sttts](https://github.com/sttts))\n\n  Courtesy of SIG API Machinery",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81872",
-    "pr_number": 81872,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81876": {
-    "commit": "c58767b1e2981bd7826b4336de0c2756fa975339",
-    "text": "Fix filter plugins are not been called during preemption",
-    "markdown": "Fix filter plugins are not been called during preemption ([#81876](https://github.com/kubernetes/kubernetes/pull/81876), [@wgliang](https://github.com/wgliang))",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81876",
-    "pr_number": 81876,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.16.0"
-  },
-  "81903": {
-    "commit": "b3b4305be5588763d39fa96d2f81a19463fb50fa",
-    "text": "kubeadm: prevent overriding of certain kubelet security configuration parameters if the user wished to modify them",
-    "markdown": "kubeadm: prevent overriding of certain kubelet security configuration parameters if the user wished to modify them ([#81903](https://github.com/kubernetes/kubernetes/pull/81903), [@jfbai](https://github.com/jfbai))",
-    "author": "jfbai",
-    "author_url": "https://github.com/jfbai",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81903",
-    "pr_number": 81903,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "81929": {
-    "commit": "3d17fd5c4f6b1ed22499c4db192e1ed6a867d579",
-    "text": "Add PodOverhead awareness to kubectl",
-    "markdown": "Add PodOverhead awareness to kubectl ([#81929](https://github.com/kubernetes/kubernetes/pull/81929), [@egernst](https://github.com/egernst))\n\n  Courtesy of SIG CLI, and SIG Scheduling",
-    "author": "egernst",
-    "author_url": "https://github.com/egernst",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81929",
-    "pr_number": 81929,
-    "areas": ["kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["cli", "scheduling"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81946": {
-    "commit": "8af33f9733d8905c76fd3db85b541943b89f67d7",
-    "text": "cache-control headers are now set appropriately.  Only openapi is cacheable if etags match.",
-    "markdown": "cache-control headers are now set appropriately.  Only openapi is cacheable if etags match. ([#81946](https://github.com/kubernetes/kubernetes/pull/81946), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81946",
-    "pr_number": 81946,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "81956": {
-    "commit": "991d71e9fe7da2035e06101f1b47cb8544592335",
-    "text": "Server-side apply is now Beta.",
-    "markdown": "Server-side apply is now Beta. ([#81956](https://github.com/kubernetes/kubernetes/pull/81956), [@apelisse](https://github.com/apelisse))\n\n  Courtesy of SIG API Machinery, SIG CLI, and SIG Testing",
-    "author": "apelisse",
-    "author_url": "https://github.com/apelisse",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81956",
-    "pr_number": 81956,
-    "areas": ["apiserver", "kubectl", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cli", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81965": {
-    "commit": "f3828b776bf270290f318c0860ab5f5c8b32b2fa",
-    "text": "The `CustomResourceValidation`, `CustomResourceSubresources`, `CustomResourceWebhookConversion` and `CustomResourcePublishOpenAPI` features are now GA, and the associated feature gates deprecated and will be removed in v1.18.",
-    "markdown": "The `CustomResourceValidation`, `CustomResourceSubresources`, `CustomResourceWebhookConversion` and `CustomResourcePublishOpenAPI` features are now GA, and the associated feature gates deprecated and will be removed in v1.18. ([#81965](https://github.com/kubernetes/kubernetes/pull/81965), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81965",
-    "pr_number": 81965,
-    "areas": ["custom-resources"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.0"
-  },
-  "81969": {
-    "commit": "7acb066dbcd7d05106609711fc4ec110fec1d1a4",
-    "text": "Adds \\livez for liveness health checking for kube-apiserver. Using the parameter `--maximum-startup-sequence-duration` will allow the liveness endpoint to defer boot-sequence failures for the specified duration period.",
-    "markdown": "Adds \\livez for liveness health checking for kube-apiserver. Using the parameter `--maximum-startup-sequence-duration` will allow the liveness endpoint to defer boot-sequence failures for the specified duration period. ([#81969](https://github.com/kubernetes/kubernetes/pull/81969), [@logicalhan](https://github.com/logicalhan))\n\n  Courtesy of SIG API Machinery, SIG Auth, and SIG Testing",
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81969",
-    "pr_number": 81969,
-    "areas": ["apiserver", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "auth", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "81972": {
-    "commit": "fe770ad612cc86eef6341945ef19bde12be50a2d",
-    "text": "- Fix disk stats in LXD using ZFS storage pool\n- Fix CRI-O missing network metris bug\n- Add `container_sockets`, `container_threads`, and `container_threads_max` metrics",
-    "markdown": "- Fix disk stats in LXD using ZFS storage pool\n  - Fix CRI-O missing network metris bug\n  - Add `container_sockets`, `container_threads`, and `container_threads_max` metrics ([#81972](https://github.com/kubernetes/kubernetes/pull/81972), [@dashpole](https://github.com/dashpole))",
-    "author": "dashpole",
-    "author_url": "https://github.com/dashpole",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81972",
-    "pr_number": 81972,
-    "areas": ["dependency"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.16.0"
-  },
-  "82004": {
-    "commit": "cd9519fe9588b66c68f6b125cbd9d776b845951c",
-    "text": "CSI ephemeral inline volume support is beta, i.e. the CSIInlineVolume feature gate is enabled by default",
-    "markdown": "CSI ephemeral inline volume support is beta, i.e. the CSIInlineVolume feature gate is enabled by default ([#82004](https://github.com/kubernetes/kubernetes/pull/82004), [@pohly](https://github.com/pohly))",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/20190122-csi-inline-volumes.md",
-        "type": "KEP"
-      },
-      {
-        "description": "[Usage]",
-        "url": "https://kubernetes.io/docs/concepts/storage/volumes/#csi-ephemeral-volumes",
-        "type": "official"
-      }
-    ],
-    "author": "pohly",
-    "author_url": "https://github.com/pohly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82004",
-    "pr_number": 82004,
-    "areas": ["test"],
-    "kinds": ["api-change"],
-    "sigs": ["storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "82005": {
-    "commit": "1698765538fada7fb51cd1965af028ed3953c9b0",
-    "text": "Add metadata.generation=1 to old CustomResources.",
-    "markdown": "Add metadata.generation=1 to old CustomResources. ([#82005](https://github.com/kubernetes/kubernetes/pull/82005), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82005",
-    "pr_number": 82005,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.0"
-  },
-  "82034": {
-    "commit": "b33968972bc178144b555189e56cb21eda15bce2",
-    "text": "Add UnschedulableAndUnresolvable status code for Scheduler Framework",
-    "markdown": "Add UnschedulableAndUnresolvable status code for Scheduler Framework ([#82034](https://github.com/kubernetes/kubernetes/pull/82034), [@alculquicondor](https://github.com/alculquicondor))\n\n  Courtesy of SIG Apps, SIG Scheduling, and SIG Testing",
-    "author": "alculquicondor",
-    "author_url": "https://github.com/alculquicondor",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82034",
-    "pr_number": 82034,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["apps", "scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "82090": {
-    "commit": "f442b6ef320140730f544527597a140e535f1e1d",
-    "text": "The apiserver now uses http/1.1 to communicate with admission webhooks, opening multiple connections to satisfy concurrent requests, and allowing spreading requests across multiple backing pods.",
-    "markdown": "The apiserver now uses http/1.1 to communicate with admission webhooks, opening multiple connections to satisfy concurrent requests, and allowing spreading requests across multiple backing pods. ([#82090](https://github.com/kubernetes/kubernetes/pull/82090), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82090",
-    "pr_number": 82090,
-    "areas": ["admission-control", "apiserver", "kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth", "node", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "82091": {
-    "commit": "1ade4ce2691a9323af79a50a0f1be3af1e7d9f1e",
-    "text": "Dual stack services (Phase II of IPv6DualStack feature) are enabled via the IPVS proxier. iptables proxier does not support dualstack yet. Dualstack iptables proxier is WIP and should catchup soon.\n\nto enable, kube-proxy must be have the following flags:\n--proxy-mode=ipvs \n--cluster-cidrs=\u003ccidr\u003e,\u003ccidr\u003e",
-    "markdown": "Dual stack services (Phase II of IPv6DualStack feature) are enabled via the IPVS proxier. iptables proxier does not support dualstack yet. Dualstack iptables proxier is WIP and should catchup soon.\n  \n  to enable, kube-proxy must be have the following flags:\n  --proxy-mode=ipvs \n  --cluster-cidrs=\u003ccidr\u003e,\u003ccidr\u003e ([#82091](https://github.com/kubernetes/kubernetes/pull/82091), [@khenidak](https://github.com/khenidak))",
-    "author": "khenidak",
-    "author_url": "https://github.com/khenidak",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82091",
-    "pr_number": 82091,
-    "areas": ["apiserver", "ipvs", "kubectl", "release-eng", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "apps", "cli", "network", "release", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "82110": {
-    "commit": "96a1558ab810064909f8a17685a55202b1f83575",
-    "text": "Graduating Windows GMSA support from alpha to beta",
-    "markdown": "Graduating Windows GMSA support from alpha to beta ([#82110](https://github.com/kubernetes/kubernetes/pull/82110), [@wk8](https://github.com/wk8))\n\n  Courtesy of SIG Windows",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-windows/20181221-windows-group-managed-service-accounts-for-container-identity.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "wk8",
-    "author_url": "https://github.com/wk8",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82110",
-    "pr_number": 82110,
-    "kinds": ["feature"],
-    "sigs": ["windows"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "82119": {
-    "commit": "82f5531df3908640a459793622ec0bd9b5bc8ea4",
-    "text": "Take the context as the first argument of Schedule.",
-    "markdown": "Take the context as the first argument of Schedule. ([#82119](https://github.com/kubernetes/kubernetes/pull/82119), [@wgliang](https://github.com/wgliang))",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82119",
-    "pr_number": 82119,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.16.0"
-  },
-  "82120": {
-    "commit": "4cc80bac919dcbb6849f31fce2cd541eb0cae002",
-    "text": "Make kubectl get --ignore-not-found continue processing when encountering error.",
-    "markdown": "Make kubectl get --ignore-not-found continue processing when encountering error. ([#82120](https://github.com/kubernetes/kubernetes/pull/82120), [@soltysh](https://github.com/soltysh))",
-    "author": "soltysh",
-    "author_url": "https://github.com/soltysh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82120",
-    "pr_number": 82120,
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.16.0"
-  },
-  "82127": {
-    "commit": "235f3ab3d920096097b213fa669508e78e31ec3e",
-    "text": "Kubeadm now includes CoreDNS version 1.6.2\n    - The CoreDNS Deployment now checks readiness via the `ready` plugin.\n    - The `proxy` plugin has been deprecated. The `forward` plugin is to be used instead.\n    - `kubernetes` plugin removes the `resyncperiod` option.\n    - The `upstream` option is deprecated and ignored if included.",
-    "markdown": "Kubeadm now includes CoreDNS version 1.6.2\n      - The CoreDNS Deployment now checks readiness via the `ready` plugin.\n      - The `proxy` plugin has been deprecated. The `forward` plugin is to be used instead.\n      - `kubernetes` plugin removes the `resyncperiod` option.\n      - The `upstream` option is deprecated and ignored if included. ([#82127](https://github.com/kubernetes/kubernetes/pull/82127), [@rajansandeep](https://github.com/rajansandeep))",
-    "author": "rajansandeep",
-    "author_url": "https://github.com/rajansandeep",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82127",
-    "pr_number": 82127,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "82140": {
-    "commit": "0466cb6e693e2846a987fe82f930f93e6e4d2a4e",
-    "text": "Node-Problem-Detector v0.7.1 is used for addon daemonset.",
-    "markdown": "Node-Problem-Detector v0.7.1 is used for addon daemonset. ([#82140](https://github.com/kubernetes/kubernetes/pull/82140), [@wangzhen127](https://github.com/wangzhen127))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Node",
-    "author": "wangzhen127",
-    "author_url": "https://github.com/wangzhen127",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82140",
-    "pr_number": 82140,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "node"],
-    "feature": true,
-    "release_version": "1.16.0"
-  },
-  "82143": {
-    "commit": "35e0f17a33e4f63b4e6328d0e0e3aabf8b6a68a6",
-    "text": "`kubectl cp` no longer supports copying symbolic links from containers; to support this use case, see `kubectl exec --help` for examples using `tar` directly",
-    "markdown": "`kubectl cp` no longer supports copying symbolic links from containers; to support this use case, see `kubectl exec --help` for examples using `tar` directly ([#82143](https://github.com/kubernetes/kubernetes/pull/82143), [@soltysh](https://github.com/soltysh))",
-    "author": "soltysh",
-    "author_url": "https://github.com/soltysh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82143",
-    "pr_number": 82143,
-    "areas": ["kubectl", "security"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.16.0"
-  },
-  "82146": {
-    "commit": "a7ac8d4c0b7fe90d51fb911511cbf6939655c877",
-    "text": "aggregated discovery requests can now timeout.  Aggregated apiserver *must- complete discovery calls within five seconds.  Other requests can take longer.\nUse feature gate `EnableAggregatedDiscoveryTimeout=false` if you *must- remove this check, but that feature gate will be removed next release.",
-    "markdown": "aggregated discovery requests can now timeout.  Aggregated apiserver *must- complete discovery calls within five seconds.  Other requests can take longer.\n  Use feature gate `EnableAggregatedDiscoveryTimeout=false` if you *must- remove this check, but that feature gate will be removed next release. ([#82146](https://github.com/kubernetes/kubernetes/pull/82146), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82146",
-    "pr_number": 82146,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.0"
-  },
-  "82170": {
-    "commit": "e3b5062057073c5800739f13a5db73073331cbd1",
-    "text": "Adding TerminationGracePeriodSeconds to the test framework API",
-    "markdown": "Adding TerminationGracePeriodSeconds to the test framework API ([#82170](https://github.com/kubernetes/kubernetes/pull/82170), [@vivekbagade](https://github.com/vivekbagade))",
-    "author": "vivekbagade",
-    "author_url": "https://github.com/vivekbagade",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82170",
-    "pr_number": 82170,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["testing"],
-    "release_version": "1.16.0"
-  },
-  "82199": {
-    "commit": "c4c64673d7746e85dd4317b05c459e81b9c49df6",
-    "text": "Updated default etcd service used by kubernetes to 3.3.15",
-    "markdown": "Updated default etcd service used by kubernetes to 3.3.15 ([#82199](https://github.com/kubernetes/kubernetes/pull/82199), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82199",
-    "pr_number": 82199,
-    "areas": ["e2e-test-framework", "kubeadm", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "82214": {
-    "commit": "f71cfdf66d0802e0f58607978f33924ffd928e8c",
-    "text": "Ensure the KUBE-MARK-DROP chain in kube-proxy mode=ipvs. The chain is ensured for both ipv4 and ipv6 in dual-stack operation.",
-    "markdown": "Ensure the KUBE-MARK-DROP chain in kube-proxy mode=ipvs. The chain is ensured for both ipv4 and ipv6 in dual-stack operation. ([#82214](https://github.com/kubernetes/kubernetes/pull/82214), [@uablrek](https://github.com/uablrek))",
-    "author": "uablrek",
-    "author_url": "https://github.com/uablrek",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82214",
-    "pr_number": 82214,
-    "areas": ["ipvs"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.16.0"
-  },
-  "82248": {
-    "commit": "c8c1aeaa5c5700f1e87524108896aa8c7f90f41a",
-    "text": "kubeadm: Allow users to skip the kube-proxy init addon phase during init and still be able to join a cluster and perform some other minor operations (but not upgrade).",
-    "markdown": "kubeadm: Allow users to skip the kube-proxy init addon phase during init and still be able to join a cluster and perform some other minor operations (but not upgrade). ([#82248](https://github.com/kubernetes/kubernetes/pull/82248), [@rosti](https://github.com/rosti))",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82248",
-    "pr_number": 82248,
-    "areas": ["kubeadm"],
-    "kinds": ["design"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "82267": {
-    "commit": "975d0736b35d966c50383f794c0114a34fbcd9e4",
-    "text": "kubeadm: fix for HTTPProxy check for IPv6 addresses (kubernetes/kubeadm\u0026#35;1769)",
-    "markdown": "kubeadm: fix for HTTPProxy check for IPv6 addresses (kubernetes/kubeadm\u0026#35;1769) ([#82267](https://github.com/kubernetes/kubernetes/pull/82267), [@kad](https://github.com/kad))",
-    "author": "kad",
-    "author_url": "https://github.com/kad",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82267",
-    "pr_number": 82267,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.0"
-  },
-  "82303": {
-    "commit": "26a381b8e1cf30021b0169721a874e77d6681b9c",
-    "text": "Fix a bug in apiserver that could cause a valid update request to be rejected with a precondition check failure.",
-    "markdown": "Fix a bug in apiserver that could cause a valid update request to be rejected with a precondition check failure. ([#82303](https://github.com/kubernetes/kubernetes/pull/82303), [@roycaihw](https://github.com/roycaihw))",
-    "author": "roycaihw",
-    "author_url": "https://github.com/roycaihw",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82303",
-    "pr_number": 82303,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.0"
-  },
-  "82314": {
-    "commit": "397ed0e8252f93484ab526b4e387f1fb4e9fc222",
-    "text": "Fix the dns suffix search list for GCE window clusters.",
-    "markdown": "Fix the dns suffix search list for GCE window clusters. ([#82314](https://github.com/kubernetes/kubernetes/pull/82314), [@lzang](https://github.com/lzang))",
-    "author": "lzang",
-    "author_url": "https://github.com/lzang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82314",
-    "pr_number": 82314,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "82322": {
-    "commit": "838e953e943dc03b05da541304733e354d29770b",
-    "text": "Bump metrics-server to v0.3.4",
-    "markdown": "Bump metrics-server to v0.3.4 ([#82322](https://github.com/kubernetes/kubernetes/pull/82322), [@olagacek](https://github.com/olagacek))",
-    "author": "olagacek",
-    "author_url": "https://github.com/olagacek",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82322",
-    "pr_number": 82322,
-    "kinds": ["bug"],
-    "sigs": ["instrumentation"],
-    "release_version": "1.16.0"
-  },
-  "82438": {
-    "commit": "e682310dcc5d805a408e0073e251d99b8fe5c06d",
-    "text": "Resolved regression serving custom resources with unhandled validation schemas with the ServerSideApply feature enabled",
-    "markdown": "Resolved regression serving custom resources with unhandled validation schemas with the ServerSideApply feature enabled ([#82438](https://github.com/kubernetes/kubernetes/pull/82438), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82438",
-    "pr_number": 82438,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "82483": {
-    "commit": "3fd8962bd1a77a3fab6b9a1bf3d5888377590d66",
-    "text": "fixed an issue that the correct PluginConfig.Args is not passed to the corresponding PluginFactory in kube-scheduler when multiple PluginConfig items are defined.",
-    "markdown": "fixed an issue that the correct PluginConfig.Args is not passed to the corresponding PluginFactory in kube-scheduler when multiple PluginConfig items are defined. ([#82483](https://github.com/kubernetes/kubernetes/pull/82483), [@everpeace](https://github.com/everpeace))",
-    "author": "everpeace",
-    "author_url": "https://github.com/everpeace",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82483",
-    "pr_number": 82483,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.16.0"
-  },
-  "82501": {
-    "commit": "6348200c92dec8848e55552f3e8039b3da95bd91",
-    "text": "Update Cluster Autoscaler to 1.16.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.16.0",
-    "markdown": "Update Cluster Autoscaler to 1.16.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.16.0 ([#82501](https://github.com/kubernetes/kubernetes/pull/82501), [@losipiuk](https://github.com/losipiuk))",
-    "author": "losipiuk",
-    "author_url": "https://github.com/losipiuk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82501",
-    "pr_number": 82501,
-    "kinds": ["bug"],
-    "sigs": ["autoscaling", "cluster-lifecycle"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "82514": {
-    "commit": "579e0c74c150085b3fac01f6a33b66db96922f93",
-    "text": "Limit the body length of exec readiness/liveness probes. remote CRIs and Docker shim read a max of 16MB output of which the exec probe itself inspects 10kb.",
-    "markdown": "Limit the body length of exec readiness/liveness probes. remote CRIs and Docker shim read a max of 16MB output of which the exec probe itself inspects 10kb. ([#82514](https://github.com/kubernetes/kubernetes/pull/82514), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82514",
-    "pr_number": 82514,
-    "areas": ["kubelet"],
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.16.0"
-  },
-  "82535": {
-    "commit": "1f2e9cfe4eb8949836886e7273e6266756617537",
-    "text": "Correct a reference to a not/no longer used kustomize subcommand in the documentation",
-    "markdown": "Correct a reference to a not/no longer used kustomize subcommand in the documentation ([#82535](https://github.com/kubernetes/kubernetes/pull/82535), [@demobox](https://github.com/demobox))",
-    "author": "demobox",
-    "author_url": "https://github.com/demobox",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82535",
-    "pr_number": 82535,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.16.0"
-  },
-  "82588": {
-    "commit": "17246b8026cebac663866e02a0fa6cdc0f766c90",
-    "text": "Fixes regression in logging spurious stack traces when proxied connections are closed by the backend",
-    "markdown": "Fixes regression in logging spurious stack traces when proxied connections are closed by the backend ([#82588](https://github.com/kubernetes/kubernetes/pull/82588), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82588",
-    "pr_number": 82588,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.0"
-  },
-  "82653": {
-    "commit": "21db9c2670984e0e4ed403a514309a53b248e3f6",
-    "text": "Restores compatibility with \u003c=1.15.x custom resources by not publishing OpenAPI for non-structural custom resource definitions",
-    "markdown": "Restores compatibility with \u003c=1.15.x custom resources by not publishing OpenAPI for non-structural custom resource definitions ([#82653](https://github.com/kubernetes/kubernetes/pull/82653), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82653",
-    "pr_number": 82653,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.0"
-  },
-  "82669": {
-    "commit": "40695b0f48b14b1b023cb063a33a1805a319fedd",
-    "text": "Single static pod files and pod files from http endpoints cannot be larger than 10 MB. HTTP probe payloads are now truncated to 10KB.",
-    "markdown": "Single static pod files and pod files from http endpoints cannot be larger than 10 MB. HTTP probe payloads are now truncated to 10KB. ([#82669](https://github.com/kubernetes/kubernetes/pull/82669), [@rphillips](https://github.com/rphillips))",
-    "author": "rphillips",
-    "author_url": "https://github.com/rphillips",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82669",
-    "pr_number": 82669,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.16.0"
-  }
-}
diff --git a/src/assets/release-notes-1.16.1.json b/src/assets/release-notes-1.16.1.json
deleted file mode 100644
index 4893c7fc..00000000
--- a/src/assets/release-notes-1.16.1.json
+++ /dev/null
@@ -1,144 +0,0 @@
-{
-  "80465": {
-    "commit": "a3b5227e0d0dc5fad565bce861e0412e2b9eb095",
-    "text": "Resolves bottleneck in internal API server communication that can cause increased goroutines and degrade API Server performance",
-    "markdown": "Resolves bottleneck in internal API server communication that can cause increased goroutines and degrade API Server performance ([#80465](https://github.com/kubernetes/kubernetes/pull/80465), [@answer1991](https://github.com/answer1991))",
-    "author": "answer1991",
-    "author_url": "https://github.com/answer1991",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80465",
-    "pr_number": 80465,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.1"
-  },
-  "82410": {
-    "commit": "85b8691a287f677aea4ded6f7a3e8bbf8bb66c5d",
-    "text": "Resolves regression generating informers for packages whose names contain `.` characters",
-    "markdown": "Resolves regression generating informers for packages whose names contain `.` characters ([#82410](https://github.com/kubernetes/kubernetes/pull/82410), [@nikhita](https://github.com/nikhita))",
-    "author": "nikhita",
-    "author_url": "https://github.com/nikhita",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82410",
-    "pr_number": 82410,
-    "areas": ["code-generation"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.1"
-  },
-  "82514": {
-    "commit": "d647ddbd755faf07169599a625faf302ffc34458",
-    "text": "Limit the body length of exec readiness/liveness probes. remote CRIs and Docker shim read a max of 16MB output of which the exec probe itself inspects 10kb.",
-    "markdown": "Limit the body length of exec readiness/liveness probes. remote CRIs and Docker shim read a max of 16MB output of which the exec probe itself inspects 10kb. ([#82514](https://github.com/kubernetes/kubernetes/pull/82514), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82514",
-    "pr_number": 82514,
-    "areas": ["kubelet"],
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.16.1"
-  },
-  "82640": {
-    "commit": "b65fd50ebb42dfd4188581bd682b7cfaf637e67c",
-    "text": "fix: azure disk detach failure if node not exists",
-    "markdown": "fix: azure disk detach failure if node not exists ([#82640](https://github.com/kubernetes/kubernetes/pull/82640), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82640",
-    "pr_number": 82640,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.1"
-  },
-  "82713": {
-    "commit": "bf6a270e43c712e31a64e9a6f9c5617c75eb26ab",
-    "text": "Resolves issue with /readyz and /livez not including etcd and kms health checks",
-    "markdown": "Resolves issue with /readyz and /livez not including etcd and kms health checks ([#82713](https://github.com/kubernetes/kubernetes/pull/82713), [@logicalhan](https://github.com/logicalhan))",
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82713",
-    "pr_number": 82713,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.1"
-  },
-  "82841": {
-    "commit": "4d90a1294810239b3f40928015f70364a6685963",
-    "text": "Fixed a scheduler panic when using PodAffinity.",
-    "markdown": "Fixed a scheduler panic when using PodAffinity. ([#82841](https://github.com/kubernetes/kubernetes/pull/82841), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82841",
-    "pr_number": 82841,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.16.1"
-  },
-  "82873": {
-    "commit": "4ad4360e9de6152b00b9fd5856b33e5e2adbb356",
-    "text": "Fix possible fd leak and closing of dirs when using openstack",
-    "markdown": "Fix possible fd leak and closing of dirs when using openstack ([#82873](https://github.com/kubernetes/kubernetes/pull/82873), [@odinuge](https://github.com/odinuge))",
-    "author": "odinuge",
-    "author_url": "https://github.com/odinuge",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82873",
-    "pr_number": 82873,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.16.1"
-  },
-  "82887": {
-    "commit": "06032937311a973ec0d82e811b14a7b6252156d8",
-    "text": "Fixes a panic in kube-controller-manager cleaning up bootstrap tokens",
-    "markdown": "Fixes a panic in kube-controller-manager cleaning up bootstrap tokens ([#82887](https://github.com/kubernetes/kubernetes/pull/82887), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82887",
-    "pr_number": 82887,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.1"
-  },
-  "83036": {
-    "commit": "40b552303be2040e42158a7724f63b70e8b7b64f",
-    "text": "Use ipv4 in wincat port forward.",
-    "markdown": "Use ipv4 in wincat port forward. ([#83036](https://github.com/kubernetes/kubernetes/pull/83036), [@liyanhui1228](https://github.com/liyanhui1228))",
-    "author": "liyanhui1228",
-    "author_url": "https://github.com/liyanhui1228",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83036",
-    "pr_number": 83036,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.16.1"
-  },
-  "83052": {
-    "commit": "05e3b64bfc472c8aa81913c35dbd3b032f71b657",
-    "text": "Update Cluster Autoscaler version to 1.16.1 (release notes: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.16.1)",
-    "markdown": "Update Cluster Autoscaler version to 1.16.1 (release notes: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.16.1) ([#83052](https://github.com/kubernetes/kubernetes/pull/83052), [@losipiuk](https://github.com/losipiuk))",
-    "author": "losipiuk",
-    "author_url": "https://github.com/losipiuk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83052",
-    "pr_number": 83052,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.1"
-  },
-  "83139": {
-    "commit": "640b5f5198a8eafefa3c02c9084630500f539c0a",
-    "text": "Update to go 1.12.10",
-    "markdown": "Update to go 1.12.10 ([#83139](https://github.com/kubernetes/kubernetes/pull/83139), [@cblecker](https://github.com/cblecker))",
-    "author": "cblecker",
-    "author_url": "https://github.com/cblecker",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83139",
-    "pr_number": 83139,
-    "areas": ["release-eng", "test"],
-    "kinds": ["bug"],
-    "sigs": ["release", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.1"
-  }
-}
diff --git a/src/assets/release-notes-1.16.2.json b/src/assets/release-notes-1.16.2.json
deleted file mode 100644
index fc2086a2..00000000
--- a/src/assets/release-notes-1.16.2.json
+++ /dev/null
@@ -1,134 +0,0 @@
-{
-  "82514": {
-    "commit": "d647ddbd755faf07169599a625faf302ffc34458",
-    "text": "Limit the body length of exec readiness/liveness probes. remote CRIs and Docker shim read a max of 16MB output of which the exec probe itself inspects 10kb.",
-    "markdown": "Limit the body length of exec readiness/liveness probes. remote CRIs and Docker shim read a max of 16MB output of which the exec probe itself inspects 10kb. ([#82514](https://github.com/kubernetes/kubernetes/pull/82514), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82514",
-    "pr_number": 82514,
-    "areas": ["kubelet"],
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.16.2"
-  },
-  "82890": {
-    "commit": "2d25f8b94e8ca0e13ae06bf1c76842c05b381974",
-    "text": "Update Azure load balancer to prevent orphaned public IP addresses",
-    "markdown": "Update Azure load balancer to prevent orphaned public IP addresses ([#82890](https://github.com/kubernetes/kubernetes/pull/82890), [@chewong](https://github.com/chewong))",
-    "author": "chewong",
-    "author_url": "https://github.com/chewong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82890",
-    "pr_number": 82890,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "cloud-provider", "network"],
-    "duplicate": true,
-    "release_version": "1.16.2"
-  },
-  "83027": {
-    "commit": "03efa96ef34ec1a044cad266d15f67fbc4145b9a",
-    "text": "Fixes kube-proxy bug accessing self nodeip:port on windows",
-    "markdown": "Fixes kube-proxy bug accessing self nodeip:port on windows ([#83027](https://github.com/kubernetes/kubernetes/pull/83027), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83027",
-    "pr_number": 83027,
-    "areas": ["dependency"],
-    "kinds": ["bug"],
-    "sigs": ["network", "windows"],
-    "duplicate": true,
-    "release_version": "1.16.2"
-  },
-  "83102": {
-    "commit": "d763cf1a209411be2f0d1df9143ac2f5731ed8e3",
-    "text": "Fix aggressive VM calls for Azure VMSS",
-    "markdown": "Fix aggressive VM calls for Azure VMSS ([#83102](https://github.com/kubernetes/kubernetes/pull/83102), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83102",
-    "pr_number": 83102,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.2"
-  },
-  "83184": {
-    "commit": "3afcd65f659e3ddb3d4124c49a7258598b147c5f",
-    "text": "Fix error where metrics related to dynamic kubelet config isn't registered",
-    "markdown": "Fix error where metrics related to dynamic kubelet config isn't registered ([#83184](https://github.com/kubernetes/kubernetes/pull/83184), [@odinuge](https://github.com/odinuge))",
-    "author": "odinuge",
-    "author_url": "https://github.com/odinuge",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83184",
-    "pr_number": 83184,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.16.2"
-  },
-  "83196": {
-    "commit": "ff564e295ff908e2aab84a3d5e34a498b1183100",
-    "text": "Use online nodes instead of possible nodes when discovering available NUMA nodes",
-    "markdown": "Use online nodes instead of possible nodes when discovering available NUMA nodes ([#83196](https://github.com/kubernetes/kubernetes/pull/83196), [@zouyee](https://github.com/zouyee))",
-    "author": "zouyee",
-    "author_url": "https://github.com/zouyee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83196",
-    "pr_number": 83196,
-    "areas": ["kubelet"],
-    "kinds": ["flake"],
-    "sigs": ["node"],
-    "release_version": "1.16.2"
-  },
-  "83261": {
-    "commit": "a2ff5d71deafff1d9688eb6f7533dd239980c420",
-    "text": "Fixes a flaw (CVE-2019-11253) in json/yaml decoding where large or malformed documents could consume excessive server resources. Request bodies for normal API requests (create/delete/update/patch operations of regular resources) are now limited to 3MB.",
-    "markdown": "Fixes a flaw (CVE-2019-11253) in json/yaml decoding where large or malformed documents could consume excessive server resources. Request bodies for normal API requests (create/delete/update/patch operations of regular resources) are now limited to 3MB. ([#83261](https://github.com/kubernetes/kubernetes/pull/83261), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83261",
-    "pr_number": 83261,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.2"
-  },
-  "83333": {
-    "commit": "f8fa26ac0b4cd49ef62edd2c7c82f8d01a9b494b",
-    "text": "Fixes a goroutine leak in kube-apiserver when a request times out.",
-    "markdown": "Fixes a goroutine leak in kube-apiserver when a request times out. ([#83333](https://github.com/kubernetes/kubernetes/pull/83333), [@lavalamp](https://github.com/lavalamp))",
-    "author": "lavalamp",
-    "author_url": "https://github.com/lavalamp",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83333",
-    "pr_number": 83333,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.2"
-  },
-  "83697": {
-    "commit": "4a8405575c56fdb97782f787272547f5cc1d4d28",
-    "text": "Fixed a bug in the single-numa-node policy of the TopologyManager.  Previously, pods that only requested CPU resources and did not request any third-party devices would fail to launch with a TopologyAffinity error. Now they will launch successfully.",
-    "markdown": "Fixed a bug in the single-numa-node policy of the TopologyManager.  Previously, pods that only requested CPU resources and did not request any third-party devices would fail to launch with a TopologyAffinity error. Now they will launch successfully. ([#83697](https://github.com/kubernetes/kubernetes/pull/83697), [@klueska](https://github.com/klueska))",
-    "author": "klueska",
-    "author_url": "https://github.com/klueska",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83697",
-    "pr_number": 83697,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.16.2"
-  },
-  "83789": {
-    "commit": "c97fe5036ef3df2967d086711e6c0c405941e14b",
-    "text": "Fixed panic when accessing CustomResources of a CRD with x-kubernetes-int-or-string.",
-    "markdown": "Fixed panic when accessing CustomResources of a CRD with x-kubernetes-int-or-string. ([#83789](https://github.com/kubernetes/kubernetes/pull/83789), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83789",
-    "pr_number": 83789,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.2"
-  }
-}
diff --git a/src/assets/release-notes-1.16.3.json b/src/assets/release-notes-1.16.3.json
deleted file mode 100644
index 824d22dc..00000000
--- a/src/assets/release-notes-1.16.3.json
+++ /dev/null
@@ -1,253 +0,0 @@
-{
-  "83427": {
-    "commit": "22f43ab27520572acaaf392817dff7b673730b05",
-    "text": "Adds a metric apiserver_request_error_total to kube-apiserver. This metric tallies the number of request_errors encountered by verb, group, version, resource, subresource, scope, component, and code.",
-    "markdown": "Adds a metric apiserver_request_error_total to kube-apiserver. This metric tallies the number of request_errors encountered by verb, group, version, resource, subresource, scope, component, and code. ([#83427](https://github.com/kubernetes/kubernetes/pull/83427), [@logicalhan](https://github.com/logicalhan))",
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83427",
-    "pr_number": 83427,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.16.3"
-  },
-  "83591": {
-    "commit": "c3aac8e00076fd2b7c1af2dc221f1c452e868de4",
-    "text": "Add data cache flushing during unmount device for GCE-PD driver in Windows Server.",
-    "markdown": "Add data cache flushing during unmount device for GCE-PD driver in Windows Server. ([#83591](https://github.com/kubernetes/kubernetes/pull/83591), [@jingxu97](https://github.com/jingxu97))",
-    "author": "jingxu97",
-    "author_url": "https://github.com/jingxu97",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83591",
-    "pr_number": 83591,
-    "kinds": ["bug"],
-    "sigs": ["storage", "windows"],
-    "duplicate": true,
-    "release_version": "1.16.3"
-  },
-  "83776": {
-    "commit": "cf477475ed3c0ab29e445f87447f6d5c25eb0ed1",
-    "text": "removed dependency on test/e2e/common from test/e2e/storage/testsuites",
-    "markdown": "removed dependency on test/e2e/common from test/e2e/storage/testsuites ([#83776](https://github.com/kubernetes/kubernetes/pull/83776), [@avalluri](https://github.com/avalluri))",
-    "author": "avalluri",
-    "author_url": "https://github.com/avalluri",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83776",
-    "pr_number": 83776,
-    "areas": ["test"],
-    "kinds": ["cleanup", "failing-test"],
-    "sigs": ["testing"],
-    "release_version": "1.16.3"
-  },
-  "83789": {
-    "commit": "c97fe5036ef3df2967d086711e6c0c405941e14b",
-    "text": "Fixed panic when accessing CustomResources of a CRD with x-kubernetes-int-or-string.",
-    "markdown": "Fixed panic when accessing CustomResources of a CRD with x-kubernetes-int-or-string. ([#83789](https://github.com/kubernetes/kubernetes/pull/83789), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83789",
-    "pr_number": 83789,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.3"
-  },
-  "83911": {
-    "commit": "a48146da9e06991ccb3d8856d78e2ae23645ecb2",
-    "text": "Fixed an issue with informers missing an `Added` event if a recently deleted object was immediately recreated at the same time the informer dropped a watch and relisted.",
-    "markdown": "Fixed an issue with informers missing an `Added` event if a recently deleted object was immediately recreated at the same time the informer dropped a watch and relisted. ([#83911](https://github.com/kubernetes/kubernetes/pull/83911), [@matte21](https://github.com/matte21))",
-    "author": "matte21",
-    "author_url": "https://github.com/matte21",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83911",
-    "pr_number": 83911,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.3"
-  },
-  "83951": {
-    "commit": "4209b9f168d0e34aeb80b2a30a5298a6a491849d",
-    "text": "Fix handling tombstones in pod-disruption-budged controller.",
-    "markdown": "Fix handling tombstones in pod-disruption-budged controller. ([#83951](https://github.com/kubernetes/kubernetes/pull/83951), [@zouyee](https://github.com/zouyee))",
-    "author": "zouyee",
-    "author_url": "https://github.com/zouyee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83951",
-    "pr_number": 83951,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.16.3"
-  },
-  "83956": {
-    "commit": "15a9f8b0b3a46017a3198980c00593d8a3b8f683",
-    "text": "Switched intstr.Type to sized integer to follow API guidelines and improve compatibility with proto libraries",
-    "markdown": "Switched intstr.Type to sized integer to follow API guidelines and improve compatibility with proto libraries ([#83956](https://github.com/kubernetes/kubernetes/pull/83956), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83956",
-    "pr_number": 83956,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.3"
-  },
-  "83968": {
-    "commit": "f2b2d8b7e2f8056d3751d6e93fb4e1c1ad6bba34",
-    "text": "Upgrade to etcd client 3.3.17 to fix bug where etcd client does not parse IPv6 addresses correctly when members are joining, and to fix bug where failover on multi-member etcd cluster fails certificate check on DNS mismatch",
-    "markdown": "Upgrade to etcd client 3.3.17 to fix bug where etcd client does not parse IPv6 addresses correctly when members are joining, and to fix bug where failover on multi-member etcd cluster fails certificate check on DNS mismatch ([#83968](https://github.com/kubernetes/kubernetes/pull/83968), [@jpbetz](https://github.com/jpbetz))",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83968",
-    "pr_number": 83968,
-    "areas": ["apiserver", "cloudprovider", "dependency"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.16.3"
-  },
-  "84038": {
-    "commit": "c1f40a5310b0abfe9a4fbddc24955360821a324b",
-    "text": "Update Cluster Autoscaler version to 1.16.2 (CA release docs: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.16.2)",
-    "markdown": "Update Cluster Autoscaler version to 1.16.2 (CA release docs: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.16.2) ([#84038](https://github.com/kubernetes/kubernetes/pull/84038), [@losipiuk](https://github.com/losipiuk))",
-    "author": "losipiuk",
-    "author_url": "https://github.com/losipiuk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84038",
-    "pr_number": 84038,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.3"
-  },
-  "84064": {
-    "commit": "e76a12b6cb1b9d77991d3fc390a87945dbcd3c15",
-    "text": "Update to use go1.12.12",
-    "markdown": "Update to use go1.12.12 ([#84064](https://github.com/kubernetes/kubernetes/pull/84064), [@cblecker](https://github.com/cblecker))",
-    "author": "cblecker",
-    "author_url": "https://github.com/cblecker",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84064",
-    "pr_number": 84064,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["release", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.3"
-  },
-  "84156": {
-    "commit": "c3f2a6524ed89bd9b3ded6f491f0d823b11523ae",
-    "text": "Fix kubelet metrics gathering on non-English Windows hosts",
-    "markdown": "Fix kubelet metrics gathering on non-English Windows hosts ([#84156](https://github.com/kubernetes/kubernetes/pull/84156), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84156",
-    "pr_number": 84156,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "windows"],
-    "duplicate": true,
-    "release_version": "1.16.3"
-  },
-  "84175": {
-    "commit": "3d9fbfdd606aaa912c6c2281bac7f3845a6cf183",
-    "text": "Fixed binding of block PersistentVolumes / PersistentVolumeClaims when BlockVolume feature is off.",
-    "markdown": "Fixed binding of block PersistentVolumes / PersistentVolumeClaims when BlockVolume feature is off. ([#84175](https://github.com/kubernetes/kubernetes/pull/84175), [@jsafrane](https://github.com/jsafrane))",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/b1b2751/keps/sig-storage/20191008-raw-block-support.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84175",
-    "pr_number": 84175,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.16.3"
-  },
-  "84223": {
-    "commit": "a7b7e162982905b46eadc55e2344bfaaa744a1ba",
-    "text": "Bumps metrics-server version to v0.3.6 for following bugfix:\n- Don't break metric storage when duplicate pod metrics encountered causing hpa to fail",
-    "markdown": "Bumps metrics-server version to v0.3.6 for following bugfix:\n  - Don't break metric storage when duplicate pod metrics encountered causing hpa to fail ([#84223](https://github.com/kubernetes/kubernetes/pull/84223), [@olagacek](https://github.com/olagacek))",
-    "author": "olagacek",
-    "author_url": "https://github.com/olagacek",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84223",
-    "pr_number": 84223,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.3"
-  },
-  "84321": {
-    "commit": "1d36d8e3e9b444f6e0b5569f7e97efc5e5ba567d",
-    "text": "CSI detach timeout increased from 10 seconds to 2 minutes",
-    "markdown": "CSI detach timeout increased from 10 seconds to 2 minutes ([#84321](https://github.com/kubernetes/kubernetes/pull/84321), [@cduchesne](https://github.com/cduchesne))",
-    "author": "cduchesne",
-    "author_url": "https://github.com/cduchesne",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84321",
-    "pr_number": 84321,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.16.3"
-  },
-  "84465": {
-    "commit": "ed185580bc39ad4316471ad2170de2c9ac0b06a2",
-    "text": "Restores compatibility of kube-scheduler with clusters that do not enable the events.k8s.io/v1beta1 API",
-    "markdown": "Restores compatibility of kube-scheduler with clusters that do not enable the events.k8s.io/v1beta1 API ([#84465](https://github.com/kubernetes/kubernetes/pull/84465), [@yastij](https://github.com/yastij))",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84465",
-    "pr_number": 84465,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.16.3"
-  },
-  "84536": {
-    "commit": "04bd01ee3f8015caf84f538b834896e03ab1ac9a",
-    "text": "azure: Add allow unsafe read from cache",
-    "markdown": "azure: Add allow unsafe read from cache ([#84536](https://github.com/kubernetes/kubernetes/pull/84536), [@aramase](https://github.com/aramase))",
-    "author": "aramase",
-    "author_url": "https://github.com/aramase",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84536",
-    "pr_number": 84536,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.3"
-  },
-  "84960": {
-    "commit": "c7dab3b96220499c9cba75285b3db61a30f79dd9",
-    "text": "kube-apiserver: fixed a bug that could cause a goroutine leak if the apiserver encountered an encoding error serving a watch to a websocket watcher",
-    "markdown": "kube-apiserver: fixed a bug that could cause a goroutine leak if the apiserver encountered an encoding error serving a watch to a websocket watcher ([#84960](https://github.com/kubernetes/kubernetes/pull/84960), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84960",
-    "pr_number": 84960,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.3"
-  },
-  "84963": {
-    "commit": "09185a04091510940228e6b63df0671aaa37606f",
-    "text": "kube-apiserver: Fixed a regression accepting patch requests \u003e 1MB",
-    "markdown": "kube-apiserver: Fixed a regression accepting patch requests \u003e 1MB ([#84963](https://github.com/kubernetes/kubernetes/pull/84963), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84963",
-    "pr_number": 84963,
-    "areas": ["apiserver", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.3"
-  },
-  "85024": {
-    "commit": "b3cbbae08ec52a7fc73d334838e18d17e8512749",
-    "text": "kubeadm: fix skipped etcd upgrade on secondary control-plane nodes when the command \"kubeadm upgrade node\" is used.",
-    "markdown": "kubeadm: fix skipped etcd upgrade on secondary control-plane nodes when the command \"kubeadm upgrade node\" is used. ([#85024](https://github.com/kubernetes/kubernetes/pull/85024), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85024",
-    "pr_number": 85024,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.3"
-  }
-}
diff --git a/src/assets/release-notes-1.16.4.json b/src/assets/release-notes-1.16.4.json
deleted file mode 100644
index eb13a11f..00000000
--- a/src/assets/release-notes-1.16.4.json
+++ /dev/null
@@ -1,171 +0,0 @@
-{
-  "84391": {
-    "commit": "5c651b7bd5f38652d278b61e289d273e5adb5936",
-    "text": "Change kube-proxy's default node IP back to 127.0.0.1, if this is incorrect, please use --bind-address to set the correct address",
-    "markdown": "Change kube-proxy's default node IP back to 127.0.0.1, if this is incorrect, please use --bind-address to set the correct address ([#84391](https://github.com/kubernetes/kubernetes/pull/84391), [@zouyee](https://github.com/zouyee))",
-    "author": "zouyee",
-    "author_url": "https://github.com/zouyee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84391",
-    "pr_number": 84391,
-    "areas": ["ipvs"],
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.16.4"
-  },
-  "84917": {
-    "commit": "02be27654e5c13539d1779eca29569135a605538",
-    "text": "fix race condition when attach/delete azure disk in same time",
-    "markdown": "fix race condition when attach/delete azure disk in same time ([#84917](https://github.com/kubernetes/kubernetes/pull/84917), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84917",
-    "pr_number": 84917,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.4"
-  },
-  "85024": {
-    "commit": "b3cbbae08ec52a7fc73d334838e18d17e8512749",
-    "text": "kubeadm: fix skipped etcd upgrade on secondary control-plane nodes when the command \"kubeadm upgrade node\" is used.",
-    "markdown": "kubeadm: fix skipped etcd upgrade on secondary control-plane nodes when the command \"kubeadm upgrade node\" is used. ([#85024](https://github.com/kubernetes/kubernetes/pull/85024), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85024",
-    "pr_number": 85024,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.4"
-  },
-  "85096": {
-    "commit": "dabc2bdb6ea75eb208653bca024c543adff8c4c8",
-    "text": "In cases where the CoreDNS migration isn't supported and the user chooses to ignore the warnings from the preflight check, the migration will be skipped and the ConfigMap and Deployment of CoreDNS will be retained.",
-    "markdown": "In cases where the CoreDNS migration isn't supported and the user chooses to ignore the warnings from the preflight check, the migration will be skipped and the ConfigMap and Deployment of CoreDNS will be retained. ([#85096](https://github.com/kubernetes/kubernetes/pull/85096), [@rajansandeep](https://github.com/rajansandeep))",
-    "author": "rajansandeep",
-    "author_url": "https://github.com/rajansandeep",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85096",
-    "pr_number": 85096,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.4"
-  },
-  "85102": {
-    "commit": "53755d662808dce0416f4106a099efc4614ded3e",
-    "text": "Change GCP ILB firewall names to contain the \"k8s-fw-\" prefix like the rest of the firewall rules. This is needed for consistency and also for other components to identify the firewall rule as k8s/service-controller managed.",
-    "markdown": "Change GCP ILB firewall names to contain the \"k8s-fw-\" prefix like the rest of the firewall rules. This is needed for consistency and also for other components to identify the firewall rule as k8s/service-controller managed. ([#85102](https://github.com/kubernetes/kubernetes/pull/85102), [@prameshj](https://github.com/prameshj))",
-    "author": "prameshj",
-    "author_url": "https://github.com/prameshj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85102",
-    "pr_number": 85102,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.4"
-  },
-  "85115": {
-    "commit": "e0a060dc5e938363ca95df69164e3d46185ed8a2",
-    "text": "azure: update disk lock logic per vm during attach/detach to allow concurrent updates for different nodes.",
-    "markdown": "azure: update disk lock logic per vm during attach/detach to allow concurrent updates for different nodes. ([#85115](https://github.com/kubernetes/kubernetes/pull/85115), [@aramase](https://github.com/aramase))",
-    "author": "aramase",
-    "author_url": "https://github.com/aramase",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85115",
-    "pr_number": 85115,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.4"
-  },
-  "85158": {
-    "commit": "81780271497f1f25686b2799fe5a63b5dd40d2d6",
-    "text": "fix vmss dirty cache issue in disk attach/detach on vmss node",
-    "markdown": "fix vmss dirty cache issue in disk attach/detach on vmss node ([#85158](https://github.com/kubernetes/kubernetes/pull/85158), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85158",
-    "pr_number": 85158,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.4"
-  },
-  "85189": {
-    "commit": "6f4915a692e42b76d4b2c1925c5dae634d6cd7ed",
-    "text": "Ensure health probes are created for local traffic policy UDP services on Azure",
-    "markdown": "Ensure health probes are created for local traffic policy UDP services on Azure ([#85189](https://github.com/kubernetes/kubernetes/pull/85189), [@nilo19](https://github.com/nilo19))",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85189",
-    "pr_number": 85189,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.4"
-  },
-  "85292": {
-    "commit": "93e538f26b87d79c4c6a19343854fc677f2c8ec3",
-    "text": "kubeadm: prevent infinite hang on \"kubeadm join\" using token discovery",
-    "markdown": "kubeadm: prevent infinite hang on \"kubeadm join\" using token discovery ([#85292](https://github.com/kubernetes/kubernetes/pull/85292), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85292",
-    "pr_number": 85292,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.4"
-  },
-  "85361": {
-    "commit": "c0f31a4ef6304d653f387455e7ed1723e7bb5385",
-    "text": "kube-controller-manager: Fixes bug setting headless service labels on endpoints",
-    "markdown": "kube-controller-manager: Fixes bug setting headless service labels on endpoints ([#85361](https://github.com/kubernetes/kubernetes/pull/85361), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85361",
-    "pr_number": 85361,
-    "kinds": ["bug"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.16.4"
-  },
-  "85385": {
-    "commit": "dca9b41efacf4d76774646243eab18d1a5426522",
-    "text": "For x-kubernetes-list-type=set a scalar or atomic item type is now required, as documented. Persisted, invalid data is tolerated.",
-    "markdown": "For x-kubernetes-list-type=set a scalar or atomic item type is now required, as documented. Persisted, invalid data is tolerated. ([#85385](https://github.com/kubernetes/kubernetes/pull/85385), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85385",
-    "pr_number": 85385,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.4"
-  },
-  "85462": {
-    "commit": "9c9944a2d59bbc4d867d325d69a38504eb1c18df",
-    "text": "Fixes a performance issue when using server-side apply with objects with very large atomic maps.",
-    "markdown": "Fixes a performance issue when using server-side apply with objects with very large atomic maps. ([#85462](https://github.com/kubernetes/kubernetes/pull/85462), [@jennybuckley](https://github.com/jennybuckley))",
-    "author": "jennybuckley",
-    "author_url": "https://github.com/jennybuckley",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85462",
-    "pr_number": 85462,
-    "areas": ["apiserver", "cloudprovider", "dependency"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cloud-provider", "scalability"],
-    "duplicate": true,
-    "release_version": "1.16.4"
-  },
-  "85865": {
-    "commit": "5f2260c4a4b206b5e832f08edec981ae55c86537",
-    "text": "Fixed issue with addon-resizer using deprecated extensions APIs",
-    "markdown": "Fixed issue with addon-resizer using deprecated extensions APIs ([#85865](https://github.com/kubernetes/kubernetes/pull/85865), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85865",
-    "pr_number": 85865,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.4"
-  }
-}
diff --git a/src/assets/release-notes-1.16.5.json b/src/assets/release-notes-1.16.5.json
deleted file mode 100644
index 82ec38a4..00000000
--- a/src/assets/release-notes-1.16.5.json
+++ /dev/null
@@ -1,758 +0,0 @@
-{
-  "73363": {
-    "commit": "67647ed5a379ea72d7db7b5461e9802aa1debdbf",
-    "text": "Minimize AWS NLB health check timeout when externalTrafficPolicy set to Local",
-    "markdown": "Minimize AWS NLB health check timeout when externalTrafficPolicy set to Local ([#73363](https://github.com/kubernetes/kubernetes/pull/73363), [@kellycampbell](https://github.com/kellycampbell))",
-    "author": "kellycampbell",
-    "author_url": "https://github.com/kellycampbell",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73363",
-    "pr_number": 73363,
-    "areas": ["cloudprovider", "provider/aws"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.5"
-  },
-  "82462": {
-    "commit": "5373fa3f593a12309ff0435858991dd117bdb6ca",
-    "text": "kube-proxy: Added dual-stack IPv4/IPv6 support to the iptables proxier.",
-    "markdown": "kube-proxy: Added dual-stack IPv4/IPv6 support to the iptables proxier. ([#82462](https://github.com/kubernetes/kubernetes/pull/82462), [@vllry](https://github.com/vllry))\n\n  Courtesy of SIG Network",
-    "author": "vllry",
-    "author_url": "https://github.com/vllry",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82462",
-    "pr_number": 82462,
-    "areas": ["ipvs"],
-    "kinds": ["feature"],
-    "sigs": ["network"],
-    "feature": true,
-    "release_version": "1.16.5"
-  },
-  "83358": {
-    "commit": "0c026656fe1c697594a756e543578ff3eab98060",
-    "text": "Bind kube-dns containers to linux nodes to avoid Windows scheduling",
-    "markdown": "Bind kube-dns containers to linux nodes to avoid Windows scheduling ([#83358](https://github.com/kubernetes/kubernetes/pull/83358), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83358",
-    "pr_number": 83358,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.16.5"
-  },
-  "83364": {
-    "commit": "fc98aa54465dbb31e2260cc51fb753c8eae55a9e",
-    "text": "Bind dns-horizontal containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes",
-    "markdown": "Bind dns-horizontal containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes ([#83364](https://github.com/kubernetes/kubernetes/pull/83364), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83364",
-    "pr_number": 83364,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.16.5"
-  },
-  "83437": {
-    "commit": "9fa1e00be9f4816e18d463f8125a7f20c0d4aee1",
-    "text": "Promote StartupProbe to beta for 1.18 release",
-    "markdown": "Promote StartupProbe to beta for 1.18 release ([#83437](https://github.com/kubernetes/kubernetes/pull/83437), [@matthyx](https://github.com/matthyx))\n\n  Courtesy of SIG Node, SIG Scalability, and SIG Testing",
-    "author": "matthyx",
-    "author_url": "https://github.com/matthyx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83437",
-    "pr_number": 83437,
-    "areas": ["test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["node", "scalability", "testing"],
-    "feature": true,
-    "release_version": "1.16.5"
-  },
-  "84163": {
-    "commit": "daaa0b9c6ba2dbf37cc0ae4464ed8b35521c7b62",
-    "text": "Bug fixes:\nMake sure we include latest packages node \u0026#35;351 (@caseydavenport)",
-    "markdown": "Bug fixes:\n  Make sure we include latest packages node \u0026#35;351 (@caseydavenport) ([#84163](https://github.com/kubernetes/kubernetes/pull/84163), [@david-tigera](https://github.com/david-tigera))",
-    "author": "david-tigera",
-    "author_url": "https://github.com/david-tigera",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84163",
-    "pr_number": 84163,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.5"
-  },
-  "84462": {
-    "commit": "a1fc96f41e943645229379a6d459826123c11c94",
-    "text": "The underlying format of the `CPUManager` state file has changed. Upgrades should be seamless, but any third-party tools that rely on reading the previous format need to be updated.",
-    "markdown": "The underlying format of the `CPUManager` state file has changed. Upgrades should be seamless, but any third-party tools that rely on reading the previous format need to be updated. ([#84462](https://github.com/kubernetes/kubernetes/pull/84462), [@klueska](https://github.com/klueska))\n\n  Courtesy of SIG Node, and SIG Testing",
-    "author": "klueska",
-    "author_url": "https://github.com/klueska",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84462",
-    "pr_number": 84462,
-    "areas": ["kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["node", "testing"],
-    "feature": true,
-    "release_version": "1.16.5"
-  },
-  "84735": {
-    "commit": "7fdefe50026895f74c7f4685ff2de90f712f05f7",
-    "text": "The sample-apiserver aggregated conformance test has updated to use the Kubernetes v1.17.0 sample apiserver",
-    "markdown": "The sample-apiserver aggregated conformance test has updated to use the Kubernetes v1.17.0 sample apiserver ([#84735](https://github.com/kubernetes/kubernetes/pull/84735), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84735",
-    "pr_number": 84735,
-    "areas": ["conformance", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "architecture", "cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.5"
-  },
-  "84951": {
-    "commit": "010291d4dc3f20d4c5f26f7e04704b9db0659b54",
-    "text": "Fixed a regression where the kubelet would fail to update the ready status of pods.",
-    "markdown": "Fixed a regression where the kubelet would fail to update the ready status of pods. ([#84951](https://github.com/kubernetes/kubernetes/pull/84951), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84951",
-    "pr_number": 84951,
-    "areas": ["kubelet"],
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.16.5"
-  },
-  "85279": {
-    "commit": "2796ff85388112d22ca0145d7762be7beb429e07",
-    "text": "New flag `--show-hidden-metrics-for-version` in kube-proxy can be used to show all hidden metrics that deprecated in the previous minor release.",
-    "markdown": "New flag `--show-hidden-metrics-for-version` in kube-proxy can be used to show all hidden metrics that deprecated in the previous minor release. ([#85279](https://github.com/kubernetes/kubernetes/pull/85279), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Network",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85279",
-    "pr_number": 85279,
-    "areas": ["kubeadm"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "feature": true,
-    "release_version": "1.16.5"
-  },
-  "85308": {
-    "commit": "a26fcf5278a7f643c3aa1dce9be60a3240bb1a72",
-    "text": "If a serving certificates param specifies a name that is an IP for an SNI certificate, it will have priority for replying to server connections.",
-    "markdown": "If a serving certificates param specifies a name that is an IP for an SNI certificate, it will have priority for replying to server connections. ([#85308](https://github.com/kubernetes/kubernetes/pull/85308), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85308",
-    "pr_number": 85308,
-    "areas": ["apiserver", "dependency"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.5"
-  },
-  "85511": {
-    "commit": "92a14f4f7ef11c580961f0aebe491d0ff148e191",
-    "text": "Fix a bug in port-forward: named port not working with service",
-    "markdown": "Fix a bug in port-forward: named port not working with service ([#85511](https://github.com/kubernetes/kubernetes/pull/85511), [@oke-py](https://github.com/oke-py))",
-    "author": "oke-py",
-    "author_url": "https://github.com/oke-py",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85511",
-    "pr_number": 85511,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.16.5"
-  },
-  "85512": {
-    "commit": "127c47caf49bec4a59a0500f99b1b1d4cdbf340e",
-    "text": "Remove cluster-monitoring addon",
-    "markdown": "Remove cluster-monitoring addon ([#85512](https://github.com/kubernetes/kubernetes/pull/85512), [@serathius](https://github.com/serathius))",
-    "author": "serathius",
-    "author_url": "https://github.com/serathius",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85512",
-    "pr_number": 85512,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["cleanup", "deprecation"],
-    "sigs": ["cluster-lifecycle", "instrumentation", "scalability", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.5"
-  },
-  "85517": {
-    "commit": "e1c711227691bd2b32519e86513bbad88b65c972",
-    "text": "add kube-proxy flags --ipvs-tcp-timeout, --ipvs-tcpfin-timeout, --ipvs-udp-timeout to configure IPVS connection timeouts.",
-    "markdown": "add kube-proxy flags --ipvs-tcp-timeout, --ipvs-tcpfin-timeout, --ipvs-udp-timeout to configure IPVS connection timeouts. ([#85517](https://github.com/kubernetes/kubernetes/pull/85517), [@andrewsykim](https://github.com/andrewsykim))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Network",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85517",
-    "pr_number": 85517,
-    "areas": ["ipvs", "kubeadm"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "feature": true,
-    "release_version": "1.16.5"
-  },
-  "85703": {
-    "commit": "d4ba10ec808aec7c3529b4b216d428e024861225",
-    "text": "Fix EndpointSlice controller race condition and ensure that it handles external changes to EndpointSlices.",
-    "markdown": "Fix EndpointSlice controller race condition and ensure that it handles external changes to EndpointSlices. ([#85703](https://github.com/kubernetes/kubernetes/pull/85703), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85703",
-    "pr_number": 85703,
-    "kinds": ["bug"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.16.5"
-  },
-  "85747": {
-    "commit": "0ab15832400c762c7203721d551d2072e2951c6d",
-    "text": "Fixes wrong validation result of NetworkPolicy PolicyTypes",
-    "markdown": "Fixes wrong validation result of NetworkPolicy PolicyTypes ([#85747](https://github.com/kubernetes/kubernetes/pull/85747), [@tnqn](https://github.com/tnqn))",
-    "author": "tnqn",
-    "author_url": "https://github.com/tnqn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85747",
-    "pr_number": 85747,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.16.5"
-  },
-  "85771": {
-    "commit": "d0e901807d4c11e725b7b3ae95d312a13d5dd7f8",
-    "text": "kubernetes will try to acquire the iptables lock every 100 msec during 5 seconds instead of every second. This specially useful for environments using kube-proxy in iptables mode with a high churn rate of services.",
-    "markdown": "kubernetes will try to acquire the iptables lock every 100 msec during 5 seconds instead of every second. This specially useful for environments using kube-proxy in iptables mode with a high churn rate of services. ([#85771](https://github.com/kubernetes/kubernetes/pull/85771), [@aojea](https://github.com/aojea))",
-    "author": "aojea",
-    "author_url": "https://github.com/aojea",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85771",
-    "pr_number": 85771,
-    "kinds": ["flake"],
-    "sigs": ["network"],
-    "release_version": "1.16.5"
-  },
-  "85789": {
-    "commit": "6db550d1db70e389463930bccfb4855f7369b815",
-    "text": "OwnerReferences []OwnerReferencematches \nso nil len(OwnerReferences) ==0 \n\nchange:\n if lease.OwnerReferences == nil || len(lease.OwnerReferences) == 0 ==\u003eif len(lease.OwnerReferences) == 0",
-    "markdown": "OwnerReferences []OwnerReferencematches \n  so nil len(OwnerReferences) ==0 \n  \n  change:\n   if lease.OwnerReferences == nil || len(lease.OwnerReferences) == 0 ==\u003eif len(lease.OwnerReferences) == 0 ([#85789](https://github.com/kubernetes/kubernetes/pull/85789), [@ZP-AlwaysWin](https://github.com/ZP-AlwaysWin))",
-    "author": "ZP-AlwaysWin",
-    "author_url": "https://github.com/ZP-AlwaysWin",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85789",
-    "pr_number": 85789,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.16.5"
-  },
-  "85836": {
-    "commit": "8727eef79c679bf40ff0cfa3318dadbc5fd50e24",
-    "text": "Add instructions about how to bring up e2e test cluster",
-    "markdown": "Add instructions about how to bring up e2e test cluster ([#85836](https://github.com/kubernetes/kubernetes/pull/85836), [@YangLu1031](https://github.com/YangLu1031))",
-    "author": "YangLu1031",
-    "author_url": "https://github.com/YangLu1031",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85836",
-    "pr_number": 85836,
-    "kinds": ["documentation"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.5"
-  },
-  "85837": {
-    "commit": "7b792c38e0c9b66413d790236c80367df3094484",
-    "text": "allow for configuration of CoreDNS replica count",
-    "markdown": "allow for configuration of CoreDNS replica count ([#85837](https://github.com/kubernetes/kubernetes/pull/85837), [@pickledrick](https://github.com/pickledrick))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "pickledrick",
-    "author_url": "https://github.com/pickledrick",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85837",
-    "pr_number": 85837,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.16.5"
-  },
-  "85879": {
-    "commit": "814fc34cde8c51ffc83870e18ad703deece63af7",
-    "text": "hollow-node use fake CRI",
-    "markdown": "hollow-node use fake CRI ([#85879](https://github.com/kubernetes/kubernetes/pull/85879), [@gongguan](https://github.com/gongguan))\n\n  Courtesy of SIG Node, and SIG Scalability",
-    "author": "gongguan",
-    "author_url": "https://github.com/gongguan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85879",
-    "pr_number": 85879,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node", "scalability"],
-    "feature": true,
-    "release_version": "1.16.5"
-  },
-  "85903": {
-    "commit": "6a919f2b54bc3fbc047250f0aa8c423f5209c2b5",
-    "text": "kube-apiserver no longer serves the following deprecated APIs:\n- All resources under `apps/v1beta1` and `apps/v1beta2` - use `apps/v1` instead\n- `daemonsets`, `deployments`, `replicasets` resources under `extensions/v1beta1` - use `apps/v1` instead\n- `networkpolicies` resources under `extensions/v1beta1` - use `networking.k8s.io/v1` instead\n- `podsecuritypolicies` resources under `extensions/v1beta1` - use `policy/v1beta1` instead",
-    "markdown": "kube-apiserver no longer serves the following deprecated APIs:\n  - All resources under `apps/v1beta1` and `apps/v1beta2` - use `apps/v1` instead\n  - `daemonsets`, `deployments`, `replicasets` resources under `extensions/v1beta1` - use `apps/v1` instead\n  - `networkpolicies` resources under `extensions/v1beta1` - use `networking.k8s.io/v1` instead\n  - `podsecuritypolicies` resources under `extensions/v1beta1` - use `policy/v1beta1` instead ([#85903](https://github.com/kubernetes/kubernetes/pull/85903), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85903",
-    "pr_number": 85903,
-    "areas": ["apiserver", "test"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["api-machinery", "apps", "cluster-lifecycle", "instrumentation", "testing"],
-    "duplicate": true,
-    "release_version": "1.16.5"
-  },
-  "85944": {
-    "commit": "a233e0115bb56e014f812365f583704075accc7b",
-    "text": "elasticsearch supports automatically setting the advertise address",
-    "markdown": "elasticsearch supports automatically setting the advertise address ([#85944](https://github.com/kubernetes/kubernetes/pull/85944), [@SataQiu](https://github.com/SataQiu))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Instrumentation",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85944",
-    "pr_number": 85944,
-    "kinds": ["bug", "feature"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "feature": true,
-    "release_version": "1.16.5"
-  },
-  "85975": {
-    "commit": "6f61072606155b070b2f79850b008685df3a74ff",
-    "text": "Fix nil pointer dereference in azure cloud provider",
-    "markdown": "Fix nil pointer dereference in azure cloud provider ([#85975](https://github.com/kubernetes/kubernetes/pull/85975), [@ldx](https://github.com/ldx))",
-    "author": "ldx",
-    "author_url": "https://github.com/ldx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85975",
-    "pr_number": 85975,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.5"
-  },
-  "85993": {
-    "commit": "7a50fdb2a635f4f3219f6411fa42d89c65a77800",
-    "text": "fix a regression in kubenet that prevent pods to obtain ip addresses",
-    "markdown": "fix a regression in kubenet that prevent pods to obtain ip addresses ([#85993](https://github.com/kubernetes/kubernetes/pull/85993), [@chendotjs](https://github.com/chendotjs))",
-    "author": "chendotjs",
-    "author_url": "https://github.com/chendotjs",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85993",
-    "pr_number": 85993,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["network", "node"],
-    "duplicate": true,
-    "release_version": "1.16.5"
-  },
-  "86016": {
-    "commit": "078344c12524adfe1f6f742e0d5a21016d65e64b",
-    "text": "Fixes kube-proxy when EndpointSlice feature gate is enabled on Windows.",
-    "markdown": "Fixes kube-proxy when EndpointSlice feature gate is enabled on Windows. ([#86016](https://github.com/kubernetes/kubernetes/pull/86016), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86016",
-    "pr_number": 86016,
-    "kinds": ["bug"],
-    "sigs": ["auth", "network"],
-    "duplicate": true,
-    "release_version": "1.16.5"
-  },
-  "86092": {
-    "commit": "442107b6b96a980e4d26c30a176071092748414e",
-    "text": "kube-proxy no longer modifies shared EndpointSlices.",
-    "markdown": "kube-proxy no longer modifies shared EndpointSlices. ([#86092](https://github.com/kubernetes/kubernetes/pull/86092), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86092",
-    "pr_number": 86092,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.16.5"
-  },
-  "86167": {
-    "commit": "03e90b80ceebc4197eae7cc6b95a59623bd9a51f",
-    "text": "change CounterVec to Counter about  PLEGDiscardEvent",
-    "markdown": "change CounterVec to Counter about  PLEGDiscardEvent ([#86167](https://github.com/kubernetes/kubernetes/pull/86167), [@yiyang5055](https://github.com/yiyang5055))",
-    "author": "yiyang5055",
-    "author_url": "https://github.com/yiyang5055",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86167",
-    "pr_number": 86167,
-    "kinds": ["cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.16.5"
-  },
-  "86168": {
-    "commit": "0d58709016f3358f34a52c12087d707a64cc6cb0",
-    "text": "Resolves performance regression in `kubectl get all` and in client-go discovery clients constructed using `NewDiscoveryClientForConfig` or `NewDiscoveryClientForConfigOrDie`.",
-    "markdown": "Resolves performance regression in `kubectl get all` and in client-go discovery clients constructed using `NewDiscoveryClientForConfig` or `NewDiscoveryClientForConfigOrDie`. ([#86168](https://github.com/kubernetes/kubernetes/pull/86168), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86168",
-    "pr_number": 86168,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.5"
-  },
-  "86210": {
-    "commit": "3ccdad175a80a73c8a756e2f35b7ac1d6a98d106",
-    "text": "The following features are unconditionally enabled and the corresponding `--feature-gates` flags have been removed: `PodPriority`, `TaintNodesByCondition`, `ResourceQuotaScopeSelectors` and `ScheduleDaemonSetPods`",
-    "markdown": "The following features are unconditionally enabled and the corresponding `--feature-gates` flags have been removed: `PodPriority`, `TaintNodesByCondition`, `ResourceQuotaScopeSelectors` and `ScheduleDaemonSetPods` ([#86210](https://github.com/kubernetes/kubernetes/pull/86210), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86210",
-    "pr_number": 86210,
-    "kinds": ["cleanup"],
-    "sigs": ["apps", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.16.5"
-  },
-  "86216": {
-    "commit": "e622579b143730b89d471ecd4e5f6761e9c37d86",
-    "text": "kubeadm: fix potential panic when executing \"kubeadm reset\" with a corrupted kubelet.conf file",
-    "markdown": "kubeadm: fix potential panic when executing \"kubeadm reset\" with a corrupted kubelet.conf file ([#86216](https://github.com/kubernetes/kubernetes/pull/86216), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86216",
-    "pr_number": 86216,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.5"
-  },
-  "86251": {
-    "commit": "49bc6966149a92e02aaee0245881c6a4a83dc007",
-    "text": "New metric kubelet_pleg_last_seen_seconds to aid diagnosis of PLEG not healthy issues.",
-    "markdown": "New metric kubelet_pleg_last_seen_seconds to aid diagnosis of PLEG not healthy issues. ([#86251](https://github.com/kubernetes/kubernetes/pull/86251), [@bboreham](https://github.com/bboreham))\n\n  Courtesy of SIG Node",
-    "author": "bboreham",
-    "author_url": "https://github.com/bboreham",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86251",
-    "pr_number": 86251,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.16.5"
-  },
-  "86266": {
-    "commit": "f06a3800ed7819fa68aa30054b4cd52619a4cbde",
-    "text": "azure cloud provider cache TTL is configurable, list of the azure cloud provider is as following:\n- \"availabilitySetNodesCacheTTLInSeconds\"\n- \"vmssCacheTTLInSeconds\"\n- \"vmssVirtualMachinesCacheTTLInSeconds\"\n- \"vmCacheTTLInSeconds\"\n- \"loadBalancerCacheTTLInSeconds\"\n- \"nsgCacheTTLInSeconds\"\n- \"routeTableCacheTTLInSeconds\"",
-    "markdown": "azure cloud provider cache TTL is configurable, list of the azure cloud provider is as following:\n  - \"availabilitySetNodesCacheTTLInSeconds\"\n  - \"vmssCacheTTLInSeconds\"\n  - \"vmssVirtualMachinesCacheTTLInSeconds\"\n  - \"vmCacheTTLInSeconds\"\n  - \"loadBalancerCacheTTLInSeconds\"\n  - \"nsgCacheTTLInSeconds\"\n  - \"routeTableCacheTTLInSeconds\" ([#86266](https://github.com/kubernetes/kubernetes/pull/86266), [@zqingqing1](https://github.com/zqingqing1))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "zqingqing1",
-    "author_url": "https://github.com/zqingqing1",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86266",
-    "pr_number": 86266,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.16.5"
-  },
-  "86277": {
-    "commit": "a57a8f33b323cd002e76c664bf0f91f71df7c927",
-    "text": "Fixed a panic in the kubelet cleaning up pod volumes",
-    "markdown": "Fixed a panic in the kubelet cleaning up pod volumes ([#86277](https://github.com/kubernetes/kubernetes/pull/86277), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86277",
-    "pr_number": 86277,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.16.5"
-  },
-  "86320": {
-    "commit": "eee9dee7b9c0ad926c8619b211902f118feff2fa",
-    "text": "Fixes an issue with kubelet-reported pod status on deleted/recreated pods.",
-    "markdown": "Fixes an issue with kubelet-reported pod status on deleted/recreated pods. ([#86320](https://github.com/kubernetes/kubernetes/pull/86320), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86320",
-    "pr_number": 86320,
-    "areas": ["kubelet"],
-    "kinds": ["bug", "flake"],
-    "sigs": ["node"],
-    "release_version": "1.16.5"
-  },
-  "86329": {
-    "commit": "4a62b3ac6dc7937ea798f2163cb3bb1f1a165f20",
-    "text": "Changed core_pattern on COS nodes to be an absolute path.",
-    "markdown": "Changed core_pattern on COS nodes to be an absolute path. ([#86329](https://github.com/kubernetes/kubernetes/pull/86329), [@mml](https://github.com/mml))",
-    "author": "mml",
-    "author_url": "https://github.com/mml",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86329",
-    "pr_number": 86329,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "node"],
-    "duplicate": true,
-    "release_version": "1.16.5"
-  },
-  "86338": {
-    "commit": "6f4ec3e0d6e1c1e9947091aedcaee882753363d6",
-    "text": "option `preConfiguredBackendPoolLoadBalancerTypes` is added to azure cloud provider for the pre-configured load balancers, possible values: `\"\"`, `\"internal\"`, \"external\"`, `\"all\"`",
-    "markdown": "option `preConfiguredBackendPoolLoadBalancerTypes` is added to azure cloud provider for the pre-configured load balancers, possible values: `\"\"`, `\"internal\"`, \"external\"`, `\"all\"` ([#86338](https://github.com/kubernetes/kubernetes/pull/86338), [@gossion](https://github.com/gossion))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "gossion",
-    "author_url": "https://github.com/gossion",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86338",
-    "pr_number": 86338,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.16.5"
-  },
-  "86351": {
-    "commit": "ac4079a5cc22ced66c868f6fd26f8e2497765e51",
-    "text": "fix: azure data disk should use same key as os disk by default",
-    "markdown": "fix: azure data disk should use same key as os disk by default ([#86351](https://github.com/kubernetes/kubernetes/pull/86351), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86351",
-    "pr_number": 86351,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.5"
-  },
-  "86369": {
-    "commit": "84198bf1206c92a14a5f237c4161d5138886492e",
-    "text": "`AlwaysCheckAllPredicates` is deprecated in scheduler Policy API.",
-    "markdown": "`AlwaysCheckAllPredicates` is deprecated in scheduler Policy API. ([#86369](https://github.com/kubernetes/kubernetes/pull/86369), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86369",
-    "pr_number": 86369,
-    "kinds": ["api-change", "cleanup", "deprecation"],
-    "sigs": ["scheduling"],
-    "release_version": "1.16.5"
-  },
-  "86412": {
-    "commit": "3577447764103cc3204b66307d651ac2be0ec83f",
-    "text": "Fixes issue where AAD token obtained by kubectl is incompatible with on-behalf-of flow and oidc.\nThe audience claim before this fix has \"spn:\" prefix. After this fix, \"spn:\" prefix is omitted.",
-    "markdown": "Fixes issue where AAD token obtained by kubectl is incompatible with on-behalf-of flow and oidc.\n  The audience claim before this fix has \"spn:\" prefix. After this fix, \"spn:\" prefix is omitted. ([#86412](https://github.com/kubernetes/kubernetes/pull/86412), [@weinong](https://github.com/weinong))",
-    "author": "weinong",
-    "author_url": "https://github.com/weinong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86412",
-    "pr_number": 86412,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.16.5"
-  },
-  "86425": {
-    "commit": "9f841bf42396daf1ed4b7ab47c1e1708bfb028b3",
-    "text": "hollow-node do not use remote CRI anymore",
-    "markdown": "hollow-node do not use remote CRI anymore ([#86425](https://github.com/kubernetes/kubernetes/pull/86425), [@jkaniuk](https://github.com/jkaniuk))",
-    "author": "jkaniuk",
-    "author_url": "https://github.com/jkaniuk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86425",
-    "pr_number": 86425,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "scalability"],
-    "duplicate": true,
-    "release_version": "1.16.5"
-  },
-  "86493": {
-    "commit": "3760d24674d9eac92284c5d19c4a774aceab1b7f",
-    "text": "kubeadm: use bind-address option to configure the kube-controller-manager and kube-scheduler http probes",
-    "markdown": "kubeadm: use bind-address option to configure the kube-controller-manager and kube-scheduler http probes ([#86493](https://github.com/kubernetes/kubernetes/pull/86493), [@aojea](https://github.com/aojea))",
-    "author": "aojea",
-    "author_url": "https://github.com/aojea",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86493",
-    "pr_number": 86493,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.5"
-  },
-  "86496": {
-    "commit": "06fb3eb582187931a22c9fc57024063d91d965f8",
-    "text": "Adds back support for AlwaysCheckAllPredicates flag.",
-    "markdown": "Adds back support for AlwaysCheckAllPredicates flag. ([#86496](https://github.com/kubernetes/kubernetes/pull/86496), [@ahg-g](https://github.com/ahg-g))",
-    "author": "ahg-g",
-    "author_url": "https://github.com/ahg-g",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86496",
-    "pr_number": 86496,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.16.5"
-  },
-  "86502": {
-    "commit": "4f70231ce7736cc748f76526c98955f86c667a41",
-    "text": "Fix issue \u0026#35;85805 about resource not found in azure cloud provider when lb specified in other resource group.",
-    "markdown": "Fix issue \u0026#35;85805 about resource not found in azure cloud provider when lb specified in other resource group. ([#86502](https://github.com/kubernetes/kubernetes/pull/86502), [@levimm](https://github.com/levimm))",
-    "author": "levimm",
-    "author_url": "https://github.com/levimm",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86502",
-    "pr_number": 86502,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.5"
-  },
-  "86515": {
-    "commit": "92e0de32698864f4dce876d79e80386261e3e18c",
-    "text": "Azure global rate limit is switched to per-client. A set of new rate limit configure options are introduced, including routeRateLimit, SubnetsRateLimit, InterfaceRateLimit, RouteTableRateLimit, LoadBalancerRateLimit, PublicIPAddressRateLimit, SecurityGroupRateLimit, VirtualMachineRateLimit, StorageAccountRateLimit, DiskRateLimit, SnapshotRateLimit, VirtualMachineScaleSetRateLimit and VirtualMachineSizeRateLimit.\n\nThe original rate limit options would be default values for those new client's rate limiter.",
-    "markdown": "Azure global rate limit is switched to per-client. A set of new rate limit configure options are introduced, including routeRateLimit, SubnetsRateLimit, InterfaceRateLimit, RouteTableRateLimit, LoadBalancerRateLimit, PublicIPAddressRateLimit, SecurityGroupRateLimit, VirtualMachineRateLimit, StorageAccountRateLimit, DiskRateLimit, SnapshotRateLimit, VirtualMachineScaleSetRateLimit and VirtualMachineSizeRateLimit.\n  \n  The original rate limit options would be default values for those new client's rate limiter. ([#86515](https://github.com/kubernetes/kubernetes/pull/86515), [@feiskyer](https://github.com/feiskyer))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86515",
-    "pr_number": 86515,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.16.5"
-  },
-  "86534": {
-    "commit": "5e31799701123c50025567b8534e1a62dbc0e9f6",
-    "text": "Fixes v1.17.0 regression in --service-cluster-ip-range handling with IPv4 ranges larger than 65536 IP addresses",
-    "markdown": "Fixes v1.17.0 regression in --service-cluster-ip-range handling with IPv4 ranges larger than 65536 IP addresses ([#86534](https://github.com/kubernetes/kubernetes/pull/86534), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86534",
-    "pr_number": 86534,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.16.5"
-  },
-  "86584": {
-    "commit": "180e1c9f5159a22576ee866a82a7dc8c26cc1c41",
-    "text": "Marked scheduler's metrics scheduling_algorithm_predicate_evaluation_seconds and\nscheduling_algorithm_priority_evaluation_seconds as deprecated. Those are replaced by framework_extension_point_duration_seconds[extenstion_point=\"Filter\"] and framework_extension_point_duration_seconds[extenstion_point=\"Score\"] respectively.",
-    "markdown": "Marked scheduler's metrics scheduling_algorithm_predicate_evaluation_seconds and\n  scheduling_algorithm_priority_evaluation_seconds as deprecated. Those are replaced by framework_extension_point_duration_seconds[extenstion_point=\"Filter\"] and framework_extension_point_duration_seconds[extenstion_point=\"Score\"] respectively. ([#86584](https://github.com/kubernetes/kubernetes/pull/86584), [@xiaoanyunfei](https://github.com/xiaoanyunfei))",
-    "author": "xiaoanyunfei",
-    "author_url": "https://github.com/xiaoanyunfei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86584",
-    "pr_number": 86584,
-    "kinds": ["cleanup", "deprecation"],
-    "sigs": ["scheduling"],
-    "release_version": "1.16.5"
-  },
-  "86586": {
-    "commit": "fdfcb00edee127df1f9d27f0b858271ed8959421",
-    "text": "Marked scheduler's scheduling_duration_seconds Summary metric as deprecated",
-    "markdown": "Marked scheduler's scheduling_duration_seconds Summary metric as deprecated ([#86586](https://github.com/kubernetes/kubernetes/pull/86586), [@xiaoanyunfei](https://github.com/xiaoanyunfei))",
-    "author": "xiaoanyunfei",
-    "author_url": "https://github.com/xiaoanyunfei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86586",
-    "pr_number": 86586,
-    "kinds": ["cleanup", "deprecation"],
-    "sigs": ["scheduling"],
-    "release_version": "1.16.5"
-  },
-  "86609": {
-    "commit": "f772c9017c16bbc983e59a19162814629a467cef",
-    "text": "kubeadm: add support for the \"ci/k8s-master\" version label as a replacement for \"ci-cross/*\", which no longer exists.",
-    "markdown": "kubeadm: add support for the \"ci/k8s-master\" version label as a replacement for \"ci-cross/*\", which no longer exists. ([#86609](https://github.com/kubernetes/kubernetes/pull/86609), [@Pensu](https://github.com/Pensu))",
-    "author": "Pensu",
-    "author_url": "https://github.com/Pensu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86609",
-    "pr_number": 86609,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.5"
-  },
-  "86612": {
-    "commit": "40320be679e9bc85632a621dc2fa12af39f37c6d",
-    "text": "fix: azure disk could not mounted on Standard_DC4s/DC2s instances",
-    "markdown": "fix: azure disk could not mounted on Standard_DC4s/DC2s instances ([#86612](https://github.com/kubernetes/kubernetes/pull/86612), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86612",
-    "pr_number": 86612,
-    "areas": ["provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.16.5"
-  },
-  "86646": {
-    "commit": "19ecd690fa1ff0747db6388714c0569dfc2548bd",
-    "text": "For subprotocol negotiation, both client and server protocol is required now.",
-    "markdown": "For subprotocol negotiation, both client and server protocol is required now. ([#86646](https://github.com/kubernetes/kubernetes/pull/86646), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86646",
-    "pr_number": 86646,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "node"],
-    "duplicate": true,
-    "release_version": "1.16.5"
-  },
-  "86693": {
-    "commit": "c72d5fd2556f90669fc3321061dba658cad077cf",
-    "text": "If a required flag is not provided to a command, the user will only see the required flag error message, instead of the entire usage menu.",
-    "markdown": "If a required flag is not provided to a command, the user will only see the required flag error message, instead of the entire usage menu. ([#86693](https://github.com/kubernetes/kubernetes/pull/86693), [@sallyom](https://github.com/sallyom))",
-    "author": "sallyom",
-    "author_url": "https://github.com/sallyom",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86693",
-    "pr_number": 86693,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.16.5"
-  },
-  "86705": {
-    "commit": "09cb73a554d3218f0267ba28de8a5fba47fd6335",
-    "text": "kubeadm: tolerate whitespace when validating certificate authority PEM data in kubeconfig files",
-    "markdown": "kubeadm: tolerate whitespace when validating certificate authority PEM data in kubeconfig files ([#86705](https://github.com/kubernetes/kubernetes/pull/86705), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86705",
-    "pr_number": 86705,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.5"
-  },
-  "86740": {
-    "commit": "b046dbce1332f6d67e8c3adf03a7b62f605b759c",
-    "text": "Azure VMSS/VMSSVM clients now suppress requests on throttling",
-    "markdown": "Azure VMSS/VMSSVM clients now suppress requests on throttling ([#86740](https://github.com/kubernetes/kubernetes/pull/86740), [@feiskyer](https://github.com/feiskyer))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86740",
-    "pr_number": 86740,
-    "areas": ["cloudprovider", "dependency", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.16.5"
-  },
-  "86940": {
-    "commit": "edad4bbfc824215fc254096dfbbd1b2ab8ce6781",
-    "text": "Add VolumeBinder method to FrameworkHandle interface, which allows user to get the volume binder when implementing scheduler framework plugins.",
-    "markdown": "Add VolumeBinder method to FrameworkHandle interface, which allows user to get the volume binder when implementing scheduler framework plugins. ([#86940](https://github.com/kubernetes/kubernetes/pull/86940), [@skilxn-go](https://github.com/skilxn-go))\n\n  Courtesy of SIG Scheduling",
-    "author": "skilxn-go",
-    "author_url": "https://github.com/skilxn-go",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86940",
-    "pr_number": 86940,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.16.5"
-  },
-  "86952": {
-    "commit": "6026a6a1515244855d10a51857af2e8c2fd26f37",
-    "text": "fix kubectl annotate error when local=true is set",
-    "markdown": "fix kubectl annotate error when local=true is set ([#86952](https://github.com/kubernetes/kubernetes/pull/86952), [@zhouya0](https://github.com/zhouya0))",
-    "author": "zhouya0",
-    "author_url": "https://github.com/zhouya0",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86952",
-    "pr_number": 86952,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.16.5"
-  },
-  "87043": {
-    "commit": "240782c7f7ee1a810207b532f970ad0fe5812939",
-    "text": "Fixed a bug which could prevent a provider ID from ever being set for node if an error occurred determining the provider ID when the node was added.",
-    "markdown": "Fixed a bug which could prevent a provider ID from ever being set for node if an error occurred determining the provider ID when the node was added. ([#87043](https://github.com/kubernetes/kubernetes/pull/87043), [@zjs](https://github.com/zjs))",
-    "author": "zjs",
-    "author_url": "https://github.com/zjs",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87043",
-    "pr_number": 87043,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.16.5"
-  }
-}
diff --git a/src/assets/release-notes-1.16.6.json b/src/assets/release-notes-1.16.6.json
deleted file mode 100644
index 17a5ed58..00000000
--- a/src/assets/release-notes-1.16.6.json
+++ /dev/null
@@ -1,253 +0,0 @@
-{
-  "76496": {
-    "commit": "f4db8212be53c69a27d893d6a4111422fbce8008",
-    "text": "The following deprecated metrics are removed, please convert to the corresponding metrics:\n1. The following replacement metrics are available from v1.14.0:\n- `rest_client_request_latency_seconds` -\u003e `rest_client_request_duration_seconds`\n- `scheduler_scheduling_latency_seconds` -\u003e `scheduler_scheduling_duration_seconds `\n- `docker_operations` -\u003e `docker_operations_total`\n- `docker_operations_latency_microseconds` -\u003e `docker_operations_duration_seconds`\n- `docker_operations_errors` -\u003e `docker_operations_errors_total`\n- `docker_operations_timeout` -\u003e `docker_operations_timeout_total`\n- `network_plugin_operations_latency_microseconds` -\u003e `network_plugin_operations_duration_seconds`\n- `kubelet_pod_worker_latency_microseconds` -\u003e `kubelet_pod_worker_duration_seconds`\n- `kubelet_pod_start_latency_microseconds` -\u003e `kubelet_pod_start_duration_seconds`\n- `kubelet_cgroup_manager_latency_microseconds` -\u003e `kubelet_cgroup_manager_duration_seconds`\n- `kubelet_pod_worker_start_latency_microseconds` -\u003e `kubelet_pod_worker_start_duration_seconds`\n- `kubelet_pleg_relist_latency_microseconds` -\u003e `kubelet_pleg_relist_duration_seconds`\n- `kubelet_pleg_relist_interval_microseconds` -\u003e `kubelet_pleg_relist_interval_seconds`\n- `kubelet_eviction_stats_age_microseconds` -\u003e `kubelet_eviction_stats_age_seconds`\n- `kubelet_runtime_operations` -\u003e `kubelet_runtime_operations_total`\n- `kubelet_runtime_operations_latency_microseconds` -\u003e `kubelet_runtime_operations_duration_seconds`\n- `kubelet_runtime_operations_errors` -\u003e `kubelet_runtime_operations_errors_total`\n- `kubelet_device_plugin_registration_count` -\u003e `kubelet_device_plugin_registration_total`\n- `kubelet_device_plugin_alloc_latency_microseconds` -\u003e `kubelet_device_plugin_alloc_duration_seconds`\n- `scheduler_e2e_scheduling_latency_microseconds` -\u003e `scheduler_e2e_scheduling_duration_seconds`\n- `scheduler_scheduling_algorithm_latency_microseconds` -\u003e `scheduler_scheduling_algorithm_duration_seconds`\n- `scheduler_scheduling_algorithm_predicate_evaluation` -\u003e `scheduler_scheduling_algorithm_predicate_evaluation_seconds`\n- `scheduler_scheduling_algorithm_priority_evaluation` -\u003e `scheduler_scheduling_algorithm_priority_evaluation_seconds`\n- `scheduler_scheduling_algorithm_preemption_evaluation` -\u003e `scheduler_scheduling_algorithm_preemption_evaluation_seconds`\n- `scheduler_binding_latency_microseconds` -\u003e `scheduler_binding_duration_seconds`\n- `kubeproxy_sync_proxy_rules_latency_microseconds` -\u003e `kubeproxy_sync_proxy_rules_duration_seconds`\n- `apiserver_request_latencies` -\u003e `apiserver_request_duration_seconds`\n- `apiserver_dropped_requests` -\u003e `apiserver_dropped_requests_total`\n- `etcd_request_latencies_summary` -\u003e `etcd_request_duration_seconds`\n- `apiserver_storage_transformation_latencies_microseconds ` -\u003e `apiserver_storage_transformation_duration_seconds`\n- `apiserver_storage_data_key_generation_latencies_microseconds` -\u003e `apiserver_storage_data_key_generation_duration_seconds`\n- `apiserver_request_count` -\u003e `apiserver_request_total`\n- `apiserver_request_latencies_summary`\n2. The following replacement metrics are available from v1.15.0:\n- `apiserver_storage_transformation_failures_total` -\u003e `apiserver_storage_transformation_operations_total`",
-    "markdown": "The following deprecated metrics are removed, please convert to the corresponding metrics:\n  1. The following replacement metrics are available from v1.14.0:\n  - `rest_client_request_latency_seconds` -\u003e `rest_client_request_duration_seconds`\n  - `scheduler_scheduling_latency_seconds` -\u003e `scheduler_scheduling_duration_seconds `\n  - `docker_operations` -\u003e `docker_operations_total`\n  - `docker_operations_latency_microseconds` -\u003e `docker_operations_duration_seconds`\n  - `docker_operations_errors` -\u003e `docker_operations_errors_total`\n  - `docker_operations_timeout` -\u003e `docker_operations_timeout_total`\n  - `network_plugin_operations_latency_microseconds` -\u003e `network_plugin_operations_duration_seconds`\n  - `kubelet_pod_worker_latency_microseconds` -\u003e `kubelet_pod_worker_duration_seconds`\n  - `kubelet_pod_start_latency_microseconds` -\u003e `kubelet_pod_start_duration_seconds`\n  - `kubelet_cgroup_manager_latency_microseconds` -\u003e `kubelet_cgroup_manager_duration_seconds`\n  - `kubelet_pod_worker_start_latency_microseconds` -\u003e `kubelet_pod_worker_start_duration_seconds`\n  - `kubelet_pleg_relist_latency_microseconds` -\u003e `kubelet_pleg_relist_duration_seconds`\n  - `kubelet_pleg_relist_interval_microseconds` -\u003e `kubelet_pleg_relist_interval_seconds`\n  - `kubelet_eviction_stats_age_microseconds` -\u003e `kubelet_eviction_stats_age_seconds`\n  - `kubelet_runtime_operations` -\u003e `kubelet_runtime_operations_total`\n  - `kubelet_runtime_operations_latency_microseconds` -\u003e `kubelet_runtime_operations_duration_seconds`\n  - `kubelet_runtime_operations_errors` -\u003e `kubelet_runtime_operations_errors_total`\n  - `kubelet_device_plugin_registration_count` -\u003e `kubelet_device_plugin_registration_total`\n  - `kubelet_device_plugin_alloc_latency_microseconds` -\u003e `kubelet_device_plugin_alloc_duration_seconds`\n  - `scheduler_e2e_scheduling_latency_microseconds` -\u003e `scheduler_e2e_scheduling_duration_seconds`\n  - `scheduler_scheduling_algorithm_latency_microseconds` -\u003e `scheduler_scheduling_algorithm_duration_seconds`\n  - `scheduler_scheduling_algorithm_predicate_evaluation` -\u003e `scheduler_scheduling_algorithm_predicate_evaluation_seconds`\n  - `scheduler_scheduling_algorithm_priority_evaluation` -\u003e `scheduler_scheduling_algorithm_priority_evaluation_seconds`\n  - `scheduler_scheduling_algorithm_preemption_evaluation` -\u003e `scheduler_scheduling_algorithm_preemption_evaluation_seconds`\n  - `scheduler_binding_latency_microseconds` -\u003e `scheduler_binding_duration_seconds`\n  - `kubeproxy_sync_proxy_rules_latency_microseconds` -\u003e `kubeproxy_sync_proxy_rules_duration_seconds`\n  - `apiserver_request_latencies` -\u003e `apiserver_request_duration_seconds`\n  - `apiserver_dropped_requests` -\u003e `apiserver_dropped_requests_total`\n  - `etcd_request_latencies_summary` -\u003e `etcd_request_duration_seconds`\n  - `apiserver_storage_transformation_latencies_microseconds ` -\u003e `apiserver_storage_transformation_duration_seconds`\n  - `apiserver_storage_data_key_generation_latencies_microseconds` -\u003e `apiserver_storage_data_key_generation_duration_seconds`\n  - `apiserver_request_count` -\u003e `apiserver_request_total`\n  - `apiserver_request_latencies_summary`\n  2. The following replacement metrics are available from v1.15.0:\n  - `apiserver_storage_transformation_failures_total` -\u003e `apiserver_storage_transformation_operations_total` ([#76496](https://github.com/kubernetes/kubernetes/pull/76496), [@danielqsj](https://github.com/danielqsj))",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76496",
-    "pr_number": 76496,
-    "areas": ["apiserver", "ipvs", "kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": [
-      "api-machinery",
-      "cluster-lifecycle",
-      "instrumentation",
-      "network",
-      "node",
-      "scheduling"
-    ],
-    "duplicate": true,
-    "release_version": "1.16.6"
-  },
-  "84265": {
-    "commit": "c9003a268dff6700506929b247847341ea4d5b33",
-    "text": "Fixed the following \n-  AWS Cloud Provider attempts to delete LoadBalancer security group it didn’t provision\n-  AWS Cloud Provider creates default LoadBalancer security group even if annotation [service.beta.kubernetes.io/aws-load-balancer-security-groups] is present",
-    "markdown": "Fixed the following \n  -  AWS Cloud Provider attempts to delete LoadBalancer security group it didn’t provision\n  -  AWS Cloud Provider creates default LoadBalancer security group even if annotation [service.beta.kubernetes.io/aws-load-balancer-security-groups] is present ([#84265](https://github.com/kubernetes/kubernetes/pull/84265), [@bhagwat070919](https://github.com/bhagwat070919))",
-    "author": "bhagwat070919",
-    "author_url": "https://github.com/bhagwat070919",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84265",
-    "pr_number": 84265,
-    "areas": ["cloudprovider", "provider/aws"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.6"
-  },
-  "85281": {
-    "commit": "d9bf22f8f7df184cf427114e8ad1fef12474364f",
-    "text": "New flag `--show-hidden-metrics-for-version` in kube-controller-manager can be used to show all hidden metrics that deprecated in the previous minor release.",
-    "markdown": "New flag `--show-hidden-metrics-for-version` in kube-controller-manager can be used to show all hidden metrics that deprecated in the previous minor release. ([#85281](https://github.com/kubernetes/kubernetes/pull/85281), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG API Machinery",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85281",
-    "pr_number": 85281,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.16.6"
-  },
-  "85283": {
-    "commit": "dd77283426bd3249004472efcbb3257744448161",
-    "text": "Cleaned up the output from `kubectl describe CSINode \u003cname\u003e`.",
-    "markdown": "Cleaned up the output from `kubectl describe CSINode \u003cname\u003e`. ([#85283](https://github.com/kubernetes/kubernetes/pull/85283), [@huffmanca](https://github.com/huffmanca))",
-    "author": "huffmanca",
-    "author_url": "https://github.com/huffmanca",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85283",
-    "pr_number": 85283,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli", "storage"],
-    "duplicate": true,
-    "release_version": "1.16.6"
-  },
-  "85458": {
-    "commit": "7f2128757b1aa2e83320d1f635fe8b2e7028542a",
-    "text": "Improved yaml parsing performance",
-    "markdown": "Improved yaml parsing performance ([#85458](https://github.com/kubernetes/kubernetes/pull/85458), [@cjcullen](https://github.com/cjcullen))",
-    "author": "cjcullen",
-    "author_url": "https://github.com/cjcullen",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85458",
-    "pr_number": 85458,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl"],
-    "kinds": ["bug"],
-    "sigs": [
-      "api-machinery",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "node"
-    ],
-    "duplicate": true,
-    "release_version": "1.16.6"
-  },
-  "85613": {
-    "commit": "05209312b74eac15f0a80577cbbdcb11bce9b8ad",
-    "text": "`kubectl apply  -f \u003cfile\u003e --prune -n \u003cnamespace\u003e` should prune all resources not defined in the file in the cli specified namespace.",
-    "markdown": "`kubectl apply  -f \u003cfile\u003e --prune -n \u003cnamespace\u003e` should prune all resources not defined in the file in the cli specified namespace. ([#85613](https://github.com/kubernetes/kubernetes/pull/85613), [@MartinKaburu](https://github.com/MartinKaburu))",
-    "author": "MartinKaburu",
-    "author_url": "https://github.com/MartinKaburu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85613",
-    "pr_number": 85613,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.16.6"
-  },
-  "85850": {
-    "commit": "61d36e4a43b831f960b190b81d371cb33b5f20d1",
-    "text": "You can now pass \"--node-ip ::\" to kubelet to indicate that it should autodetect an IPv6 address to use as the node's primary address.",
-    "markdown": "You can now pass \"--node-ip ::\" to kubelet to indicate that it should autodetect an IPv6 address to use as the node's primary address. ([#85850](https://github.com/kubernetes/kubernetes/pull/85850), [@danwinship](https://github.com/danwinship))\n\n  Courtesy of SIG Cloud Provider, SIG Network, and SIG Node",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85850",
-    "pr_number": 85850,
-    "areas": ["ipv6", "kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider", "network", "node"],
-    "feature": true,
-    "release_version": "1.16.6"
-  },
-  "86282": {
-    "commit": "be26fbc638ff5ed952ccae70e68baa712b3c60c8",
-    "text": "kubelet: resource metrics endpoint `/metrics/resource/v1alpha1` as well as all metrics under this endpoint have been deprecated.\nPlease convert to the following metrics emitted by endpoint `/metrics/resource`:\n- scrape_error --\u003e scrape_error\n- node_cpu_usage_seconds_total --\u003e node_cpu_usage_seconds\n- node_memory_working_set_bytes --\u003e node_memory_working_set_bytes\n- container_cpu_usage_seconds_total --\u003e container_cpu_usage_seconds\n- container_memory_working_set_bytes --\u003e container_memory_working_set_bytes\n- scrape_error --\u003e scrape_error",
-    "markdown": "kubelet: resource metrics endpoint `/metrics/resource/v1alpha1` as well as all metrics under this endpoint have been deprecated.\n  Please convert to the following metrics emitted by endpoint `/metrics/resource`:\n  - scrape_error --\u003e scrape_error\n  - node_cpu_usage_seconds_total --\u003e node_cpu_usage_seconds\n  - node_memory_working_set_bytes --\u003e node_memory_working_set_bytes\n  - container_cpu_usage_seconds_total --\u003e container_cpu_usage_seconds\n  - container_memory_working_set_bytes --\u003e container_memory_working_set_bytes\n  - scrape_error --\u003e scrape_error ([#86282](https://github.com/kubernetes/kubernetes/pull/86282), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Node",
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86282",
-    "pr_number": 86282,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.16.6"
-  },
-  "86861": {
-    "commit": "916edd922e528f2f27542d3ca5967ca11fe153f8",
-    "text": "kubeadm upgrades always persist the etcd backup for stacked",
-    "markdown": "kubeadm upgrades always persist the etcd backup for stacked ([#86861](https://github.com/kubernetes/kubernetes/pull/86861), [@SataQiu](https://github.com/SataQiu))",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86861",
-    "pr_number": 86861,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.16.6"
-  },
-  "86899": {
-    "commit": "3e8155ead3d890697141d4946ec5fdd5b0dab192",
-    "text": "kubeadm: support automatic retry after failing to pull image",
-    "markdown": "kubeadm: support automatic retry after failing to pull image ([#86899](https://github.com/kubernetes/kubernetes/pull/86899), [@SataQiu](https://github.com/SataQiu))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86899",
-    "pr_number": 86899,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.16.6"
-  },
-  "87029": {
-    "commit": "af46f2cdd2a72ef808f99715c978d58a4603871b",
-    "text": "Fixes service account token admission error in clusters that do not run the service account token controller",
-    "markdown": "Fixes service account token admission error in clusters that do not run the service account token controller ([#87029](https://github.com/kubernetes/kubernetes/pull/87029), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87029",
-    "pr_number": 87029,
-    "kinds": ["bug"],
-    "sigs": ["auth"],
-    "release_version": "1.16.6"
-  },
-  "87035": {
-    "commit": "d2a526bc0eb0961966584749935e7a28bd769adc",
-    "text": "provider/azure: Network security groups can now be in a separate resource group.",
-    "markdown": "provider/azure: Network security groups can now be in a separate resource group. ([#87035](https://github.com/kubernetes/kubernetes/pull/87035), [@CecileRobertMichon](https://github.com/CecileRobertMichon))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "CecileRobertMichon",
-    "author_url": "https://github.com/CecileRobertMichon",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87035",
-    "pr_number": 87035,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.16.6"
-  },
-  "87044": {
-    "commit": "36e40fb850293076b415ae3d376f5f81dc897105",
-    "text": "TODO",
-    "markdown": "TODO ([#87044](https://github.com/kubernetes/kubernetes/pull/87044), [@jennybuckley](https://github.com/jennybuckley))\n\n  Courtesy of SIG API Machinery",
-    "author": "jennybuckley",
-    "author_url": "https://github.com/jennybuckley",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87044",
-    "pr_number": 87044,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.16.6"
-  },
-  "87122": {
-    "commit": "93595eb4eb933bfed90b1f4a1f5597caccf7ccfd",
-    "text": "Azure network and VM clients now suppress requests on throttling",
-    "markdown": "Azure network and VM clients now suppress requests on throttling ([#87122](https://github.com/kubernetes/kubernetes/pull/87122), [@feiskyer](https://github.com/feiskyer))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87122",
-    "pr_number": 87122,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.16.6"
-  },
-  "87203": {
-    "commit": "0c8dcc59f099f69240ffd83eb2eae1cb6e1c68aa",
-    "text": "All apiservers log request lines in a more greppable format.",
-    "markdown": "All apiservers log request lines in a more greppable format. ([#87203](https://github.com/kubernetes/kubernetes/pull/87203), [@lavalamp](https://github.com/lavalamp))",
-    "author": "lavalamp",
-    "author_url": "https://github.com/lavalamp",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87203",
-    "pr_number": 87203,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.16.6"
-  },
-  "87213": {
-    "commit": "a25c6aa35067187800ef2a0928646f5d23268764",
-    "text": "CustomResourceDefinition status fields are no longer required for client validation when submitting manifests.",
-    "markdown": "CustomResourceDefinition status fields are no longer required for client validation when submitting manifests. ([#87213](https://github.com/kubernetes/kubernetes/pull/87213), [@hasheddan](https://github.com/hasheddan))\n\n  Courtesy of SIG API Machinery",
-    "author": "hasheddan",
-    "author_url": "https://github.com/hasheddan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87213",
-    "pr_number": 87213,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.16.6"
-  },
-  "87246": {
-    "commit": "6413f1ee2be99fe5ba18844a2646fb15d462bec4",
-    "text": "Fix the bug PIP's DNS is deleted if no DNS label service annotation isn't set.",
-    "markdown": "Fix the bug PIP's DNS is deleted if no DNS label service annotation isn't set. ([#87246](https://github.com/kubernetes/kubernetes/pull/87246), [@nilo19](https://github.com/nilo19))",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87246",
-    "pr_number": 87246,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.16.6"
-  }
-}
diff --git a/src/assets/release-notes-1.17.0.json b/src/assets/release-notes-1.17.0.json
deleted file mode 100644
index cc7bc3fa..00000000
--- a/src/assets/release-notes-1.17.0.json
+++ /dev/null
@@ -1,4374 +0,0 @@
-{
-  "71653": {
-    "commit": "605687dec7ae3842641930fc451d43876f5424f7",
-    "text": "The docker container runtime now enforces a 220 second timeout on container network operations.",
-    "markdown": "The docker container runtime now enforces a 220 second timeout on container network operations. ([#71653](https://github.com/kubernetes/kubernetes/pull/71653), [@liucimin](https://github.com/liucimin))",
-    "author": "liucimin",
-    "author_url": "https://github.com/liucimin",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/71653",
-    "pr_number": 71653,
-    "areas": ["dependency", "kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["network", "node"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "72046": {
-    "commit": "d9be37e926b97c1687abef74953df4844f013972",
-    "text": "Support Service Topology",
-    "markdown": "Support Service Topology ([#72046](https://github.com/kubernetes/kubernetes/pull/72046), [@m1093782566](https://github.com/m1093782566))\n\n  Courtesy of SIG Apps, SIG Auth, and SIG Network",
-    "author": "m1093782566",
-    "author_url": "https://github.com/m1093782566",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72046",
-    "pr_number": 72046,
-    "areas": ["ipvs"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["apps", "auth", "network"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "72170": {
-    "commit": "54d49bd785ba6fe8eb651e8a73ce3580f8c0ec41",
-    "text": "The resource version option, when passed to a list call, is now consistently interpreted as the minimum allowed resource version.  Previously when listing resources that had the watch cache disabled clients could retrieve a snapshot at that exact resource version.  If the client requests a resource version newer than the current state, a TimeoutError is returned suggesting the client retry in a few seconds.  This behavior is now consistent for both single item retrieval and list calls, and for when the watch cache is enabled or disabled.",
-    "markdown": "The resource version option, when passed to a list call, is now consistently interpreted as the minimum allowed resource version.  Previously when listing resources that had the watch cache disabled clients could retrieve a snapshot at that exact resource version.  If the client requests a resource version newer than the current state, a TimeoutError is returned suggesting the client retry in a few seconds.  This behavior is now consistent for both single item retrieval and list calls, and for when the watch cache is enabled or disabled. ([#72170](https://github.com/kubernetes/kubernetes/pull/72170), [@jpbetz](https://github.com/jpbetz))",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/72170",
-    "pr_number": 72170,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "74026": {
-    "commit": "cb2684c41671f78563f8e92b6c5fafbf45471534",
-    "text": "A node that uses a CSI raw block volume needs to be drained before kubelet can be upgraded to 1.17.",
-    "markdown": "A node that uses a CSI raw block volume needs to be drained before kubelet can be upgraded to 1.17. ([#74026](https://github.com/kubernetes/kubernetes/pull/74026), [@mkimuram](https://github.com/mkimuram))",
-    "author": "mkimuram",
-    "author_url": "https://github.com/mkimuram",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74026",
-    "pr_number": 74026,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node", "storage"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "75588": {
-    "commit": "0a08798b9a6867ee32c5555b3a887a47ec70d10e",
-    "text": "Ceph RBD volume plugin now does not use any keyring (/etc/ceph/ceph.client.lvs01cinder.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin) for authentication. Ceph user credentials must be provided in PersistentVolume objects and referred Secrets.",
-    "markdown": "Ceph RBD volume plugin now does not use any keyring (/etc/ceph/ceph.client.lvs01cinder.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin) for authentication. Ceph user credentials must be provided in PersistentVolume objects and referred Secrets. ([#75588](https://github.com/kubernetes/kubernetes/pull/75588), [@smileusd](https://github.com/smileusd))",
-    "author": "smileusd",
-    "author_url": "https://github.com/smileusd",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/75588",
-    "pr_number": 75588,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.17.0"
-  },
-  "76310": {
-    "commit": "bb55aa7c54c327fc0d71671e978e7626ce366497",
-    "text": "Critical pods can now be created in namespaces other than kube-system. To limit critical pods to the kube-system namespace, cluster admins should create an admission configuration file limiting critical pods by default, and a matching quota object in the `kube-system` namespace permitting critical pods in that namespace. See https://kubernetes.io/docs/concepts/policy/resource-quotas/\u0026#35;limit-priority-class-consumption-by-default for details.",
-    "markdown": "Critical pods can now be created in namespaces other than kube-system. To limit critical pods to the kube-system namespace, cluster admins should create an admission configuration file limiting critical pods by default, and a matching quota object in the `kube-system` namespace permitting critical pods in that namespace. See https://kubernetes.io/docs/concepts/policy/resource-quotas/\u0026#35;limit-priority-class-consumption-by-default for details. ([#76310](https://github.com/kubernetes/kubernetes/pull/76310), [@ravisantoshgudimetla](https://github.com/ravisantoshgudimetla))",
-    "author": "ravisantoshgudimetla",
-    "author_url": "https://github.com/ravisantoshgudimetla",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76310",
-    "pr_number": 76310,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "76471": {
-    "commit": "39929d82d0f03416e291195e583678cbd3f3e0ad",
-    "text": "A new `--prefix` flag added into `kubectl logs` which prepends each log line with information about it's source (pod name and container name)",
-    "markdown": "A new `--prefix` flag added into `kubectl logs` which prepends each log line with information about it's source (pod name and container name) ([#76471](https://github.com/kubernetes/kubernetes/pull/76471), [@m1kola](https://github.com/m1kola))\n\n  Courtesy of SIG CLI",
-    "author": "m1kola",
-    "author_url": "https://github.com/m1kola",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76471",
-    "pr_number": 76471,
-    "areas": ["kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "78764": {
-    "commit": "a7e0b768ee7e87b3dd8106b482c89e51bb2518a2",
-    "text": "kubeadm: implemented structured output of 'kubeadm token list' in JSON, YAML, Go template and JsonPath formats",
-    "markdown": "kubeadm: implemented structured output of 'kubeadm token list' in JSON, YAML, Go template and JsonPath formats ([#78764](https://github.com/kubernetes/kubernetes/pull/78764), [@bart0sh](https://github.com/bart0sh))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "bart0sh",
-    "author_url": "https://github.com/bart0sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/78764",
-    "pr_number": 78764,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "79223": {
-    "commit": "eedfb6bc7a2e3e4d3d69a9fc466dd58795fb0283",
-    "text": "kubeadm no longer defaults or validates the component configs of the kubelet or kube-proxy",
-    "markdown": "kubeadm no longer defaults or validates the component configs of the kubelet or kube-proxy ([#79223](https://github.com/kubernetes/kubernetes/pull/79223), [@rosti](https://github.com/rosti))",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79223",
-    "pr_number": 79223,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "79993": {
-    "commit": "ded6ee953c68f8333ee6291e0bcb7e58604fac00",
-    "text": "kube-controller-manager\n--node-cidr-mask-size-ipv4 int32     Default: 24. Mask size for IPv4 node-cidr in dual-stack cluster.\n--node-cidr-mask-size-ipv6 int32     Default: 64. Mask size for IPv6 node-cidr in dual-stack cluster.\n\nThese 2 flags can be used only for dual-stack clusters. For non dual-stack clusters, continue to use\n--node-cidr-mask-size flag to configure the mask size.\n\nThe default node cidr mask size for IPv6 was 24 which is now changed to 64.",
-    "markdown": "kube-controller-manager\n  --node-cidr-mask-size-ipv4 int32     Default: 24. Mask size for IPv4 node-cidr in dual-stack cluster.\n  --node-cidr-mask-size-ipv6 int32     Default: 64. Mask size for IPv6 node-cidr in dual-stack cluster.\n  \n  These 2 flags can be used only for dual-stack clusters. For non dual-stack clusters, continue to use\n  --node-cidr-mask-size flag to configure the mask size.\n  \n  The default node cidr mask size for IPv6 was 24 which is now changed to 64. ([#79993](https://github.com/kubernetes/kubernetes/pull/79993), [@aramase](https://github.com/aramase))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Cloud Provider, SIG Network, and SIG Testing",
-    "author": "aramase",
-    "author_url": "https://github.com/aramase",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/79993",
-    "pr_number": 79993,
-    "areas": ["test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "cloud-provider", "network", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "80004": {
-    "commit": "e129a6bc3f39eea5cdadb4d3ed2bccd548ab8920",
-    "text": "When scaling down a ReplicaSet, delete doubled up replicas first, where a \"doubled up replica\" is defined as one that is on the same node as an active replica belonging to a related ReplicaSet.  ReplicaSets are considered \"related\" if they have a common controller (typically a Deployment).",
-    "markdown": "When scaling down a ReplicaSet, delete doubled up replicas first, where a \"doubled up replica\" is defined as one that is on the same node as an active replica belonging to a related ReplicaSet.  ReplicaSets are considered \"related\" if they have a common controller (typically a Deployment). ([#80004](https://github.com/kubernetes/kubernetes/pull/80004), [@Miciah](https://github.com/Miciah))\n\n  Courtesy of SIG Apps, SIG Autoscaling, SIG Scalability, and SIG Testing",
-    "author": "Miciah",
-    "author_url": "https://github.com/Miciah",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80004",
-    "pr_number": 80004,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["api-change", "bug", "feature"],
-    "sigs": ["apps", "autoscaling", "scalability", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "80058": {
-    "commit": "805eecf6922a78a7f1e145f347413aa185b2a942",
-    "text": "Enables VolumeSnapshotDataSource feature gate and promotes volume snapshot APIs to beta.",
-    "markdown": "Enables VolumeSnapshotDataSource feature gate and promotes volume snapshot APIs to beta. ([#80058](https://github.com/kubernetes/kubernetes/pull/80058), [@xing-yang](https://github.com/xing-yang))\n\n  Courtesy of SIG Cluster Lifecycle, SIG Storage, and SIG Testing",
-    "author": "xing-yang",
-    "author_url": "https://github.com/xing-yang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80058",
-    "pr_number": 80058,
-    "areas": ["test"],
-    "kinds": ["bug", "feature"],
-    "sigs": ["cluster-lifecycle", "storage", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "80284": {
-    "commit": "48ddf3be87789c92e6824c9ce536c76d008f5c19",
-    "text": "Fix a racing issue in client-go UpdateTransportConfig.",
-    "markdown": "Fix a racing issue in client-go UpdateTransportConfig. ([#80284](https://github.com/kubernetes/kubernetes/pull/80284), [@danielqsj](https://github.com/danielqsj))",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80284",
-    "pr_number": 80284,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "80458": {
-    "commit": "2b4dc315a352a0677e1ced03557cd03087cfad68",
-    "text": "Fixes the bug in informer-gen that it produces incorrect code if a type has nonNamespaced tag set.",
-    "markdown": "Fixes the bug in informer-gen that it produces incorrect code if a type has nonNamespaced tag set. ([#80458](https://github.com/kubernetes/kubernetes/pull/80458), [@tatsuhiro-t](https://github.com/tatsuhiro-t))",
-    "author": "tatsuhiro-t",
-    "author_url": "https://github.com/tatsuhiro-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80458",
-    "pr_number": 80458,
-    "areas": ["code-generation"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "80465": {
-    "commit": "45f7f70479424a04029e4c437cff1d54385d6c39",
-    "text": "Resolves bottleneck in internal API server communication that can cause increased goroutines and degrade API Server performance",
-    "markdown": "Resolves bottleneck in internal API server communication that can cause increased goroutines and degrade API Server performance ([#80465](https://github.com/kubernetes/kubernetes/pull/80465), [@answer1991](https://github.com/answer1991))",
-    "author": "answer1991",
-    "author_url": "https://github.com/answer1991",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80465",
-    "pr_number": 80465,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "80572": {
-    "commit": "a6f51da5001e82d965906c46239589cbcba086f0",
-    "text": "Scale custom resource unconditionally if resourceVersion is not provided",
-    "markdown": "Scale custom resource unconditionally if resourceVersion is not provided ([#80572](https://github.com/kubernetes/kubernetes/pull/80572), [@knight42](https://github.com/knight42))",
-    "author": "knight42",
-    "author_url": "https://github.com/knight42",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80572",
-    "pr_number": 80572,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "80747": {
-    "commit": "9f036cd43d35a9c41d7ac4ca82398a6d0bef957b",
-    "text": "On AWS nodes with multiple network interfaces, kubelet should now more reliably report the same primary node IP.",
-    "markdown": "On AWS nodes with multiple network interfaces, kubelet should now more reliably report the same primary node IP. ([#80747](https://github.com/kubernetes/kubernetes/pull/80747), [@danwinship](https://github.com/danwinship))",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80747",
-    "pr_number": 80747,
-    "areas": ["cloudprovider", "provider/aws"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "network", "node"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "80854": {
-    "commit": "09d142a7ab093faa8a1efc49e37e45058b4a915a",
-    "text": "Kubenet: added HostPort IPv6 support\nHostPortManager: operates only with one IP family, failing if receives portmapping entries with different IP families\nHostPortSyncer: operates only with one IP family, skipping portmap entries with different IP families",
-    "markdown": "Kubenet: added HostPort IPv6 support\n  HostPortManager: operates only with one IP family, failing if receives portmapping entries with different IP families\n  HostPortSyncer: operates only with one IP family, skipping portmap entries with different IP families ([#80854](https://github.com/kubernetes/kubernetes/pull/80854), [@aojea](https://github.com/aojea))\n\n  Courtesy of SIG Network, and SIG Node",
-    "author": "aojea",
-    "author_url": "https://github.com/aojea",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80854",
-    "pr_number": 80854,
-    "areas": ["kubelet"],
-    "kinds": ["bug", "feature"],
-    "sigs": ["network", "node"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "80880": {
-    "commit": "1e94c12d416dfb1dcb4bd06631298f592abcadfa",
-    "text": "Fix validation message to mention bytes, not characters.",
-    "markdown": "Fix validation message to mention bytes, not characters. ([#80880](https://github.com/kubernetes/kubernetes/pull/80880), [@DirectXMan12](https://github.com/DirectXMan12))",
-    "author": "DirectXMan12",
-    "author_url": "https://github.com/DirectXMan12",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80880",
-    "pr_number": 80880,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "80883": {
-    "commit": "3edbc6afff17ea8dfe5c10b2677dcdc8767f67e2",
-    "text": "The mutating and validating admission webhook plugins now read configuration from the admissionregistration.k8s.io/v1 API.",
-    "markdown": "The mutating and validating admission webhook plugins now read configuration from the admissionregistration.k8s.io/v1 API. ([#80883](https://github.com/kubernetes/kubernetes/pull/80883), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80883",
-    "pr_number": 80883,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "81158": {
-    "commit": "1f5d329b4827919937a20e6fec1302668b68ef8e",
-    "text": "Fix unsafe JSON construction in a number of locations in the codebase",
-    "markdown": "Fix unsafe JSON construction in a number of locations in the codebase ([#81158](https://github.com/kubernetes/kubernetes/pull/81158), [@zouyee](https://github.com/zouyee))",
-    "author": "zouyee",
-    "author_url": "https://github.com/zouyee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81158",
-    "pr_number": 81158,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "apps", "node"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "81263": {
-    "commit": "b8b7c376edb62ce8be42cef3728112c5c71d9fa4",
-    "text": "Add \"podInitialBackoffDurationSeconds\" and \"podMaxBackoffDurationSeconds\" to the scheduler config API",
-    "markdown": "Add \"podInitialBackoffDurationSeconds\" and \"podMaxBackoffDurationSeconds\" to the scheduler config API ([#81263](https://github.com/kubernetes/kubernetes/pull/81263), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Apps, and SIG Scheduling",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81263",
-    "pr_number": 81263,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["apps", "scheduling"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "81280": {
-    "commit": "4cff1c3ea1b2478d9139bccb7018750a022ee5f0",
-    "text": "If container fails because ContainerCannotRun, do not utilize the FallbackToLogsOnError TerminationMessagePolicy, as it masks more useful logs.",
-    "markdown": "If container fails because ContainerCannotRun, do not utilize the FallbackToLogsOnError TerminationMessagePolicy, as it masks more useful logs. ([#81280](https://github.com/kubernetes/kubernetes/pull/81280), [@yqwang-ms](https://github.com/yqwang-ms))",
-    "author": "yqwang-ms",
-    "author_url": "https://github.com/yqwang-ms",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81280",
-    "pr_number": 81280,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.17.0"
-  },
-  "81319": {
-    "commit": "2bc3804a0e492aa21a368bde054f150edda2a346",
-    "text": "kubeadm: add a upgrade health check that deploys a Job",
-    "markdown": "kubeadm: add a upgrade health check that deploys a Job ([#81319](https://github.com/kubernetes/kubernetes/pull/81319), [@neolit123](https://github.com/neolit123))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81319",
-    "pr_number": 81319,
-    "areas": ["kubeadm", "security"],
-    "kinds": ["cleanup", "feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "81344": {
-    "commit": "17a57f99d59c8e32d5539538523f91f97f4934a7",
-    "text": "sourcesReady provides the readiness of kubelet configuration sources such as apiserver update readiness.",
-    "markdown": "sourcesReady provides the readiness of kubelet configuration sources such as apiserver update readiness. ([#81344](https://github.com/kubernetes/kubernetes/pull/81344), [@zouyee](https://github.com/zouyee))",
-    "author": "zouyee",
-    "author_url": "https://github.com/zouyee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81344",
-    "pr_number": 81344,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "node"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "81431": {
-    "commit": "3ee195f0e9b691557d6f2c8b8b2009c6acbbf93f",
-    "text": "Deprecate the beta labels for zones (\"failure-domain.beta.kubernetes.io/zone\") and \nregions (\"failure-domain.beta.kubernetes.io/region\") in favor of their GA equivalents:\n\"topology.kubernetes.io/zone\" and \"topology.kubernetes.io/region\". \n\nThe beta labels \"failure-domain.beta.kubernetes.io/zone\" and \"failure-domain.beta.kubernetes.io/region\" will be removed in v1.21",
-    "markdown": "Deprecate the beta labels for zones (\"failure-domain.beta.kubernetes.io/zone\") and \n  regions (\"failure-domain.beta.kubernetes.io/region\") in favor of their GA equivalents:\n  \"topology.kubernetes.io/zone\" and \"topology.kubernetes.io/region\". \n  \n  The beta labels \"failure-domain.beta.kubernetes.io/zone\" and \"failure-domain.beta.kubernetes.io/region\" will be removed in v1.21 ([#81431](https://github.com/kubernetes/kubernetes/pull/81431), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81431",
-    "pr_number": 81431,
-    "areas": ["cloudprovider", "e2e-test-framework", "kubelet", "test"],
-    "kinds": ["api-change"],
-    "sigs": [
-      "apps",
-      "auth",
-      "cloud-provider",
-      "network",
-      "node",
-      "scheduling",
-      "storage",
-      "testing"
-    ],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "81517": {
-    "commit": "e7090e8f5e0decfc03899330824386c69af274d4",
-    "text": "Kubernetes no longer monitors firewalld. On systems using firewalld for firewall\nmaintenance, kube-proxy will take slightly longer to recover from disruptive\nfirewalld operations that delete kube-proxy's iptables rules.\n\nAs a side effect of these changes, kube-proxy's\n`sync_proxy_rules_last_timestamp_seconds` metric no longer behaves the\nway it used to; now it will only change when services or endpoints actually\nchange, rather than reliably updating every 60 seconds (or whatever). If you\nare trying to monitor for whether iptables updates are failing, the\n`sync_proxy_rules_iptables_restore_failures_total` metric may be more useful.",
-    "markdown": "Kubernetes no longer monitors firewalld. On systems using firewalld for firewall\n  maintenance, kube-proxy will take slightly longer to recover from disruptive\n  firewalld operations that delete kube-proxy's iptables rules.\n  \n  As a side effect of these changes, kube-proxy's\n  `sync_proxy_rules_last_timestamp_seconds` metric no longer behaves the\n  way it used to; now it will only change when services or endpoints actually\n  change, rather than reliably updating every 60 seconds (or whatever). If you\n  are trying to monitor for whether iptables updates are failing, the\n  `sync_proxy_rules_iptables_restore_failures_total` metric may be more useful. ([#81517](https://github.com/kubernetes/kubernetes/pull/81517), [@danwinship](https://github.com/danwinship))",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81517",
-    "pr_number": 81517,
-    "areas": ["dependency", "kube-proxy", "kubeadm", "kubelet", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "network", "node", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "81670": {
-    "commit": "eb42df3e9cdb522578845c7476a3708ba7002ee0",
-    "text": "The example API server has renamed its `wardle.k8s.io` API group to `wardle.example.com`",
-    "markdown": "The example API server has renamed its `wardle.k8s.io` API group to `wardle.example.com` ([#81670](https://github.com/kubernetes/kubernetes/pull/81670), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81670",
-    "pr_number": 81670,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "81971": {
-    "commit": "3ebee370c5d65661bc59fa3c10be4919b5f94bf1",
-    "text": "The kubectl's api-resource command now has a `--sort-by` flag to sort resources by name or kind.",
-    "markdown": "The kubectl's api-resource command now has a `--sort-by` flag to sort resources by name or kind. ([#81971](https://github.com/kubernetes/kubernetes/pull/81971), [@laddng](https://github.com/laddng))\n\n  Courtesy of SIG CLI",
-    "author": "laddng",
-    "author_url": "https://github.com/laddng",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81971",
-    "pr_number": 81971,
-    "areas": ["kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "82049": {
-    "commit": "9cf309ed59bfda30d0f9279120db1121367bf5da",
-    "text": "Deprecate the instance type beta label (\"beta.kubernetes.io/instance-type\") in favor of it's GA equivalent: \"node.kubernetes.io/instance-type\"",
-    "markdown": "Deprecate the instance type beta label (\"beta.kubernetes.io/instance-type\") in favor of it's GA equivalent: \"node.kubernetes.io/instance-type\" ([#82049](https://github.com/kubernetes/kubernetes/pull/82049), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82049",
-    "pr_number": 82049,
-    "areas": ["kubelet"],
-    "kinds": ["api-change"],
-    "sigs": ["apps", "auth", "cloud-provider", "node", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "82148": {
-    "commit": "5675ef858ef1039bdd5d7b8a97c71fb8da5bd581",
-    "text": "k8s dockerconfigjson secrets are now compatible with docker config desktop authentication credentials files",
-    "markdown": "k8s dockerconfigjson secrets are now compatible with docker config desktop authentication credentials files ([#82148](https://github.com/kubernetes/kubernetes/pull/82148), [@bbourbie](https://github.com/bbourbie))",
-    "author": "bbourbie",
-    "author_url": "https://github.com/bbourbie",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82148",
-    "pr_number": 82148,
-    "kinds": ["bug"],
-    "sigs": ["auth", "node"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "82176": {
-    "commit": "b140b431073ae4d84ce9ef5e01a1f27058178ead",
-    "text": "more complete and accurate logging of stack backtraces in E2E failures",
-    "markdown": "more complete and accurate logging of stack backtraces in E2E failures ([#82176](https://github.com/kubernetes/kubernetes/pull/82176), [@pohly](https://github.com/pohly))",
-    "author": "pohly",
-    "author_url": "https://github.com/pohly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82176",
-    "pr_number": 82176,
-    "areas": ["apiserver", "cloudprovider", "dependency", "e2e-test-framework", "kubectl", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "82264": {
-    "commit": "8ad7e78d37fbd1ce42d75056ea9c00a7f63aefcc",
-    "text": "Openstack: Do not delete managed LB in case of security group reconciliation errors",
-    "markdown": "Openstack: Do not delete managed LB in case of security group reconciliation errors ([#82264](https://github.com/kubernetes/kubernetes/pull/82264), [@multi-io](https://github.com/multi-io))",
-    "author": "multi-io",
-    "author_url": "https://github.com/multi-io",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82264",
-    "pr_number": 82264,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.17.0"
-  },
-  "82347": {
-    "commit": "129a13af983fe2b6bd4429c47e5f0cde6bf0ca42",
-    "text": "kube-dns add-on: \n- All containers are now being executed under more restrictive privileges. \n- Most of the containers now run as non-root user and has the root filesystem set as read-only. \n- The remaining container running as root only has the minimum Linux capabilities it requires to run. \n- Privilege escalation has been disabled for all containers.",
-    "markdown": "kube-dns add-on: \n  - All containers are now being executed under more restrictive privileges. \n  - Most of the containers now run as non-root user and has the root filesystem set as read-only. \n  - The remaining container running as root only has the minimum Linux capabilities it requires to run. \n  - Privilege escalation has been disabled for all containers. ([#82347](https://github.com/kubernetes/kubernetes/pull/82347), [@pjbgf](https://github.com/pjbgf))",
-    "author": "pjbgf",
-    "author_url": "https://github.com/pjbgf",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82347",
-    "pr_number": 82347,
-    "areas": ["security"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "82409": {
-    "commit": "07025a5d9ed1e697e765f725b5e9023199c41ad1",
-    "text": "Added metrics 'authentication_latency_seconds' that can be used to understand the latency of authentication.",
-    "markdown": "Added metrics 'authentication_latency_seconds' that can be used to understand the latency of authentication. ([#82409](https://github.com/kubernetes/kubernetes/pull/82409), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG API Machinery, SIG Auth, and SIG Instrumentation",
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82409",
-    "pr_number": 82409,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "auth", "instrumentation"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "82410": {
-    "commit": "973629f742e600eb5762e4c57126a73c5b3cb500",
-    "text": "Resolves regression generating informers for packages whose names contain `.` characters",
-    "markdown": "Resolves regression generating informers for packages whose names contain `.` characters ([#82410](https://github.com/kubernetes/kubernetes/pull/82410), [@nikhita](https://github.com/nikhita))",
-    "author": "nikhita",
-    "author_url": "https://github.com/nikhita",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82410",
-    "pr_number": 82410,
-    "areas": ["code-generation"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "82423": {
-    "commit": "ea4f7853bb24a6c3fe7aeeee1aa89f8273c8db64",
-    "text": "If a bad flag is supplied to a kubectl command, only a tip to run --help is printed, instead of the usage menu.  Usage menu is printed upon running `kubectl command --help`.",
-    "markdown": "If a bad flag is supplied to a kubectl command, only a tip to run --help is printed, instead of the usage menu.  Usage menu is printed upon running `kubectl command --help`. ([#82423](https://github.com/kubernetes/kubernetes/pull/82423), [@sallyom](https://github.com/sallyom))",
-    "author": "sallyom",
-    "author_url": "https://github.com/sallyom",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82423",
-    "pr_number": 82423,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.17.0"
-  },
-  "82432": {
-    "commit": "f2ee75cfa29210d82dd50e440dcadfa0a23e6159",
-    "text": "Expose kubernetes client in the scheduling framework handle.",
-    "markdown": "Expose kubernetes client in the scheduling framework handle. ([#82432](https://github.com/kubernetes/kubernetes/pull/82432), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Scheduling",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82432",
-    "pr_number": 82432,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "82472": {
-    "commit": "3a19f1e80b172dfceb06ffe654b1e349bac53f73",
-    "text": "The deprecated feature gates `GCERegionalPersistentDisk`, `EnableAggregatedDiscoveryTimeout` and `PersistentLocalVolumes` are now unconditionally enabled and can no longer be specified in component invocations.",
-    "markdown": "The deprecated feature gates `GCERegionalPersistentDisk`, `EnableAggregatedDiscoveryTimeout` and `PersistentLocalVolumes` are now unconditionally enabled and can no longer be specified in component invocations. ([#82472](https://github.com/kubernetes/kubernetes/pull/82472), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG API Machinery, SIG Storage, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82472",
-    "pr_number": 82472,
-    "areas": ["apiserver", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "storage", "testing"],
-    "action_required": true,
-    "release_version": "1.17.0"
-  },
-  "82473": {
-    "commit": "f3981710db8c6e90567e91f4c0d68e8122418290",
-    "text": "kubeadm: use the --service-cluster-ip-range flag to init or use the ServiceSubnet field in the kubeadm config to pass a comma separated list of Service CIDRs.",
-    "markdown": "kubeadm: use the --service-cluster-ip-range flag to init or use the ServiceSubnet field in the kubeadm config to pass a comma separated list of Service CIDRs. ([#82473](https://github.com/kubernetes/kubernetes/pull/82473), [@Arvinderpal](https://github.com/Arvinderpal))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "Arvinderpal",
-    "author_url": "https://github.com/Arvinderpal",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82473",
-    "pr_number": 82473,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "82496": {
-    "commit": "29a52f524114379f3e57917cb11c1a672d5917f4",
-    "text": "Reduces the number of calls made to the Azure API when requesting the instance view of a virtual machine scale set node.",
-    "markdown": "Reduces the number of calls made to the Azure API when requesting the instance view of a virtual machine scale set node. ([#82496](https://github.com/kubernetes/kubernetes/pull/82496), [@hasheddan](https://github.com/hasheddan))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "hasheddan",
-    "author_url": "https://github.com/hasheddan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82496",
-    "pr_number": 82496,
-    "areas": ["cloudprovider", "dependency"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "82508": {
-    "commit": "cfa3e2c499d5316ccf9f837e9bc8238cd6aeaff5",
-    "text": "Fix panic in kubelet when running IPv4/IPv6 dual-stack mode with a CNI plugin",
-    "markdown": "Fix panic in kubelet when running IPv4/IPv6 dual-stack mode with a CNI plugin ([#82508](https://github.com/kubernetes/kubernetes/pull/82508), [@aanm](https://github.com/aanm))",
-    "author": "aanm",
-    "author_url": "https://github.com/aanm",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82508",
-    "pr_number": 82508,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["network", "node"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "82511": {
-    "commit": "dd5efc420b80df2fac8a550e6e05a57a11807c50",
-    "text": "Fix typos in `certificates.k8s.io/v1beta1` KeyUsage constant names: `UsageContentCommittment` becomes `UsageContentCommitment` and `UsageNetscapSGC` becomes `UsageNetscapeSGC`.",
-    "markdown": "Fix typos in `certificates.k8s.io/v1beta1` KeyUsage constant names: `UsageContentCommittment` becomes `UsageContentCommitment` and `UsageNetscapSGC` becomes `UsageNetscapeSGC`. ([#82511](https://github.com/kubernetes/kubernetes/pull/82511), [@abursavich](https://github.com/abursavich))",
-    "author": "abursavich",
-    "author_url": "https://github.com/abursavich",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82511",
-    "pr_number": 82511,
-    "areas": ["kubeadm"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["auth", "cluster-lifecycle"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "82574": {
-    "commit": "24b1e7f27f6df30969e297038200e82a473d812f",
-    "text": "Added cloud operation count metrics to azure cloud controller manager.",
-    "markdown": "Added cloud operation count metrics to azure cloud controller manager. ([#82574](https://github.com/kubernetes/kubernetes/pull/82574), [@kkmsft](https://github.com/kkmsft))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "kkmsft",
-    "author_url": "https://github.com/kkmsft",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82574",
-    "pr_number": 82574,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "82578": {
-    "commit": "cb1799bd489029f696014343892ee55084f61266",
-    "text": "The VolumeSubpathEnvExpansion feature is graduating to GA. The `VolumeSubpathEnvExpansion` feature gate is unconditionally enabled, and will be removed in v1.19.",
-    "markdown": "The VolumeSubpathEnvExpansion feature is graduating to GA. The `VolumeSubpathEnvExpansion` feature gate is unconditionally enabled, and will be removed in v1.19. ([#82578](https://github.com/kubernetes/kubernetes/pull/82578), [@kevtaylor](https://github.com/kevtaylor))",
-    "author": "kevtaylor",
-    "author_url": "https://github.com/kevtaylor",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82578",
-    "pr_number": 82578,
-    "areas": ["test"],
-    "kinds": ["api-change"],
-    "sigs": ["apps", "node", "storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "82603": {
-    "commit": "c2513c8ae30cf1d5a7d0b9a8b76c360760191b8e",
-    "text": "Fixed kubectl endpointslice output for get requests",
-    "markdown": "Fixed kubectl endpointslice output for get requests ([#82603](https://github.com/kubernetes/kubernetes/pull/82603), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82603",
-    "pr_number": 82603,
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.17.0"
-  },
-  "82616": {
-    "commit": "4fb75e2f0d9a36c47edcf65f89bb92f20274ee56",
-    "text": "kubeadm: when adding extra apiserver authorization-modes, the defaults \"Node,RBAC\" are no longer prepended in the resulting static Pod manifests and a full override is allowed.",
-    "markdown": "kubeadm: when adding extra apiserver authorization-modes, the defaults \"Node,RBAC\" are no longer prepended in the resulting static Pod manifests and a full override is allowed. ([#82616](https://github.com/kubernetes/kubernetes/pull/82616), [@ghouscht](https://github.com/ghouscht))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "ghouscht",
-    "author_url": "https://github.com/ghouscht",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82616",
-    "pr_number": 82616,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "action_required": true,
-    "release_version": "1.17.0"
-  },
-  "82640": {
-    "commit": "ba07527278ef2cde9c27886ec3333cfef472112a",
-    "text": "fix: azure disk detach failure if node not exists",
-    "markdown": "fix: azure disk detach failure if node not exists ([#82640](https://github.com/kubernetes/kubernetes/pull/82640), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82640",
-    "pr_number": 82640,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.17.0"
-  },
-  "82643": {
-    "commit": "dd8a23ca2e366ccd247179f13da372762d81b33e",
-    "text": "\u0026#35; kubectl rollout history sts/test-sts \nstatefulset.apps/test-sts \nREVISION\n0\n0\n0\n1\n2\n3",
-    "markdown": "\u0026#35; kubectl rollout history sts/test-sts \n  statefulset.apps/test-sts \n  REVISION\n  0\n  0\n  0\n  1\n  2\n  3 ([#82643](https://github.com/kubernetes/kubernetes/pull/82643), [@ZP-AlwaysWin](https://github.com/ZP-AlwaysWin))",
-    "author": "ZP-AlwaysWin",
-    "author_url": "https://github.com/ZP-AlwaysWin",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82643",
-    "pr_number": 82643,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.17.0"
-  },
-  "82660": {
-    "commit": "646afd52ff93b069401a61dc3b4b9ec37a1d936a",
-    "text": "kubectl drain node --dry-run will list pods that would be evicted or deleted",
-    "markdown": "kubectl drain node --dry-run will list pods that would be evicted or deleted ([#82660](https://github.com/kubernetes/kubernetes/pull/82660), [@sallyom](https://github.com/sallyom))",
-    "author": "sallyom",
-    "author_url": "https://github.com/sallyom",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82660",
-    "pr_number": 82660,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.17.0"
-  },
-  "82664": {
-    "commit": "4568c615269860433f914e1dc74b13e66acbe5ef",
-    "text": "Conformance tests may now include disruptive tests. If you are running tests against a live cluster, consider skipping those tests tagged as `Disruptive` to avoid non-test workloads being impacted. Be aware, skipping any conformance tests (even disruptive ones) will make the results ineligible for consideration for the CNCF Certified Kubernetes program.",
-    "markdown": "Conformance tests may now include disruptive tests. If you are running tests against a live cluster, consider skipping those tests tagged as `Disruptive` to avoid non-test workloads being impacted. Be aware, skipping any conformance tests (even disruptive ones) will make the results ineligible for consideration for the CNCF Certified Kubernetes program. ([#82664](https://github.com/kubernetes/kubernetes/pull/82664), [@johnSchnake](https://github.com/johnSchnake))",
-    "author": "johnSchnake",
-    "author_url": "https://github.com/johnSchnake",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82664",
-    "pr_number": 82664,
-    "areas": ["conformance", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["architecture", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "82690": {
-    "commit": "b32a3d9ca4c94e1105050edcd27456bb3751498c",
-    "text": "The ResourceQuotaScopeSelectors feature has graduated to GA. The `ResourceQuotaScopeSelectors` feature gate is now unconditionally enabled and will be removed in 1.18.",
-    "markdown": "The ResourceQuotaScopeSelectors feature has graduated to GA. The `ResourceQuotaScopeSelectors` feature gate is now unconditionally enabled and will be removed in 1.18. ([#82690](https://github.com/kubernetes/kubernetes/pull/82690), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Scheduling",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-scheduling/20190409-resource-quota-ga.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82690",
-    "pr_number": 82690,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "82703": {
-    "commit": "aab740ffc2444dd91ab59a034fdb490370b9b1cc",
-    "text": "Graduate TaintNodesByCondition to GA in 1.17. (feature gate will be removed in 1.18) action required",
-    "markdown": "Graduate TaintNodesByCondition to GA in 1.17. (feature gate will be removed in 1.18) action required ([#82703](https://github.com/kubernetes/kubernetes/pull/82703), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Cloud Provider, SIG Node, SIG Scheduling, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82703",
-    "pr_number": 82703,
-    "areas": ["apiserver", "kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "apps", "cloud-provider", "node", "scheduling", "testing"],
-    "feature": true,
-    "action_required": true,
-    "release_version": "1.17.0"
-  },
-  "82705": {
-    "commit": "a5fe905be420d518892e8e8b682902deef82d1c6",
-    "text": "configmaps/extension-apiserver-authentication in kube-system is continuously updated by kube-apiservers, instead of just at apiserver start",
-    "markdown": "configmaps/extension-apiserver-authentication in kube-system is continuously updated by kube-apiservers, instead of just at apiserver start ([#82705](https://github.com/kubernetes/kubernetes/pull/82705), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82705",
-    "pr_number": 82705,
-    "areas": ["apiserver", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "82707": {
-    "commit": "3ef3b6a5baf9e57bfda739dcb06cb0e1c882fb98",
-    "text": "- When registering with a 1.17+ API server, MutatingWebhookConfiguration and ValidatingWebhookConfiguration objects can now request that only `v1` AdmissionReview requests be sent to them. Previously, webhooks were required to support receiving `v1beta1` AdmissionReview requests as well for compatibility with API servers \u003c= 1.15.\n- When registering with a 1.17+ API server, a CustomResourceDefinition conversion webhook can now request that only `v1` ConversionReview requests be sent to them. Previously, conversion webhooks were required to support receiving `v1beta1` ConversionReview requests as well for compatibility with API servers \u003c= 1.15.",
-    "markdown": "- When registering with a 1.17+ API server, MutatingWebhookConfiguration and ValidatingWebhookConfiguration objects can now request that only `v1` AdmissionReview requests be sent to them. Previously, webhooks were required to support receiving `v1beta1` AdmissionReview requests as well for compatibility with API servers \u003c= 1.15.\n  - When registering with a 1.17+ API server, a CustomResourceDefinition conversion webhook can now request that only `v1` ConversionReview requests be sent to them. Previously, conversion webhooks were required to support receiving `v1beta1` ConversionReview requests as well for compatibility with API servers \u003c= 1.15. ([#82707](https://github.com/kubernetes/kubernetes/pull/82707), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82707",
-    "pr_number": 82707,
-    "kinds": ["api-change", "cleanup", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "82713": {
-    "commit": "61affb33267391042cc07e2679cb733925ace20a",
-    "text": "Resolves issue with /readyz and /livez not including etcd and kms health checks",
-    "markdown": "Resolves issue with /readyz and /livez not including etcd and kms health checks ([#82713](https://github.com/kubernetes/kubernetes/pull/82713), [@logicalhan](https://github.com/logicalhan))",
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82713",
-    "pr_number": 82713,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "82759": {
-    "commit": "caeda2112aba9a054ae9c8fae03b1708d48bee5d",
-    "text": "Report non-confusing error for negative storage size in PVC spec.",
-    "markdown": "Report non-confusing error for negative storage size in PVC spec. ([#82759](https://github.com/kubernetes/kubernetes/pull/82759), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82759",
-    "pr_number": 82759,
-    "kinds": ["bug"],
-    "sigs": ["apps", "storage"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "82795": {
-    "commit": "7dc298af423f521451d6dc34b9ba803c8b46daae",
-    "text": "Graduate ScheduleDaemonSetPods to GA. (feature gate will be removed in 1.18) action required.",
-    "markdown": "Graduate ScheduleDaemonSetPods to GA. (feature gate will be removed in 1.18) action required. ([#82795](https://github.com/kubernetes/kubernetes/pull/82795), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Apps, SIG Scheduling, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82795",
-    "pr_number": 82795,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["apps", "scheduling", "testing"],
-    "feature": true,
-    "action_required": true,
-    "release_version": "1.17.0"
-  },
-  "82809": {
-    "commit": "695c3061dd92a6b6950f8adf0341ceb4a8dd44d7",
-    "text": "Kubernetes now requires go1.13.4+ to build",
-    "markdown": "Kubernetes now requires go1.13.4+ to build ([#82809](https://github.com/kubernetes/kubernetes/pull/82809), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82809",
-    "pr_number": 82809,
-    "areas": [
-      "apiserver",
-      "cloudprovider",
-      "code-generation",
-      "dependency",
-      "ipvs",
-      "kubeadm",
-      "kubectl",
-      "kubelet",
-      "release-eng",
-      "test"
-    ],
-    "kinds": ["cleanup"],
-    "sigs": [
-      "api-machinery",
-      "auth",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "network",
-      "node",
-      "release",
-      "storage",
-      "testing"
-    ],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "82830": {
-    "commit": "ac8ac0fc170eec7cdf292d841f8d026a65ce1681",
-    "text": "PersistentVolumeLabel admission plugin, responsible for labeling `PersistentVolumes` with topology labels, now does not overwrite existing labels on PVs that were dynamically provisioned. It trusts the  dynamic provisioning that it provided the correct labels to the `PersistentVolume`, saving one potentially expensive cloud API call. `PersistentVolumes` created manually by users are labelled by the admission plugin in the same way as before.",
-    "markdown": "PersistentVolumeLabel admission plugin, responsible for labeling `PersistentVolumes` with topology labels, now does not overwrite existing labels on PVs that were dynamically provisioned. It trusts the  dynamic provisioning that it provided the correct labels to the `PersistentVolume`, saving one potentially expensive cloud API call. `PersistentVolumes` created manually by users are labelled by the admission plugin in the same way as before. ([#82830](https://github.com/kubernetes/kubernetes/pull/82830), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82830",
-    "pr_number": 82830,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.17.0"
-  },
-  "82841": {
-    "commit": "33adc7fafa68ad3b91c55dfa4e3db0ce807ad5ab",
-    "text": "Fixed a scheduler panic when using PodAffinity.",
-    "markdown": "Fixed a scheduler panic when using PodAffinity. ([#82841](https://github.com/kubernetes/kubernetes/pull/82841), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82841",
-    "pr_number": 82841,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.17.0"
-  },
-  "82842": {
-    "commit": "b0e5b9c7b06b56c5d444c934ab7503d10f4d5a1b",
-    "text": "Modified the scheduling framework's Filter API.",
-    "markdown": "Modified the scheduling framework's Filter API. ([#82842](https://github.com/kubernetes/kubernetes/pull/82842), [@ahg-g](https://github.com/ahg-g))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "ahg-g",
-    "author_url": "https://github.com/ahg-g",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82842",
-    "pr_number": 82842,
-    "areas": ["test"],
-    "kinds": ["bug", "feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "82856": {
-    "commit": "b281315450d1882aeb9ccf2ecdc888626b89b50a",
-    "text": "Update crictl to v1.16.1.",
-    "markdown": "Update crictl to v1.16.1. ([#82856](https://github.com/kubernetes/kubernetes/pull/82856), [@Random-Liu](https://github.com/Random-Liu))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Node",
-    "author": "Random-Liu",
-    "author_url": "https://github.com/Random-Liu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82856",
-    "pr_number": 82856,
-    "kinds": ["bug", "feature"],
-    "sigs": ["cluster-lifecycle", "node"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "82873": {
-    "commit": "338a09abd26a7b549cb9ba17655f9fe1c0c10f44",
-    "text": "Fix possible fd leak and closing of dirs when using openstack",
-    "markdown": "Fix possible fd leak and closing of dirs when using openstack ([#82873](https://github.com/kubernetes/kubernetes/pull/82873), [@odinuge](https://github.com/odinuge))",
-    "author": "odinuge",
-    "author_url": "https://github.com/odinuge",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82873",
-    "pr_number": 82873,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "82887": {
-    "commit": "73505056fb757140da6616d61e905222ef4fe939",
-    "text": "Fixes a panic in kube-controller-manager cleaning up bootstrap tokens",
-    "markdown": "Fixes a panic in kube-controller-manager cleaning up bootstrap tokens ([#82887](https://github.com/kubernetes/kubernetes/pull/82887), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82887",
-    "pr_number": 82887,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "82890": {
-    "commit": "e89023cff364a4af92a0abcfdfbf47ca7819d1a5",
-    "text": "Update Azure load balancer to prevent orphaned public IP addresses",
-    "markdown": "Update Azure load balancer to prevent orphaned public IP addresses ([#82890](https://github.com/kubernetes/kubernetes/pull/82890), [@chewong](https://github.com/chewong))",
-    "author": "chewong",
-    "author_url": "https://github.com/chewong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82890",
-    "pr_number": 82890,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "cloud-provider", "network"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "82912": {
-    "commit": "0b4cccc9d12af5a097b0d13f101566cbcfd9895f",
-    "text": "New APIs to allow adding/removing pods from pre-calculated prefilter state in the scheduling framework",
-    "markdown": "New APIs to allow adding/removing pods from pre-calculated prefilter state in the scheduling framework ([#82912](https://github.com/kubernetes/kubernetes/pull/82912), [@ahg-g](https://github.com/ahg-g))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "ahg-g",
-    "author_url": "https://github.com/ahg-g",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82912",
-    "pr_number": 82912,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "82927": {
-    "commit": "0ff761b14f26dc59815f63c72dc00d2f69f4f7b8",
-    "text": "kube-proxy: a configuration file specified via `--config` is now loaded with strict deserialization, which fails if the config file contains duplicate or unknown fields. This protects against accidentally running with config files that are malformed, mis-indented, or have typos in field names, and getting unexpected behavior.",
-    "markdown": "kube-proxy: a configuration file specified via `--config` is now loaded with strict deserialization, which fails if the config file contains duplicate or unknown fields. This protects against accidentally running with config files that are malformed, mis-indented, or have typos in field names, and getting unexpected behavior. ([#82927](https://github.com/kubernetes/kubernetes/pull/82927), [@obitech](https://github.com/obitech))\n\n  Courtesy of SIG API Machinery, SIG Cluster Lifecycle, and SIG Network",
-    "author": "obitech",
-    "author_url": "https://github.com/obitech",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82927",
-    "pr_number": 82927,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "network"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "82951": {
-    "commit": "05aaeac8b2b4a4c9a625c11648fc72e5cab07f84",
-    "text": "Added Clone method to the scheduling framework's PluginContext and ContextData.",
-    "markdown": "Added Clone method to the scheduling framework's PluginContext and ContextData. ([#82951](https://github.com/kubernetes/kubernetes/pull/82951), [@ahg-g](https://github.com/ahg-g))\n\n  Courtesy of SIG Scheduling",
-    "author": "ahg-g",
-    "author_url": "https://github.com/ahg-g",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82951",
-    "pr_number": 82951,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "82966": {
-    "commit": "be4683ef9cc68f50f0f970f036167d84b85d8ed3",
-    "text": "The official kube-proxy image (used by kubeadm, among other things) is now\ncompatible with systems running iptables 1.8 in \"nft\" mode, and will autodetect\nwhich mode it should use.",
-    "markdown": "The official kube-proxy image (used by kubeadm, among other things) is now\n  compatible with systems running iptables 1.8 in \"nft\" mode, and will autodetect\n  which mode it should use. ([#82966](https://github.com/kubernetes/kubernetes/pull/82966), [@danwinship](https://github.com/danwinship))\n\n  Courtesy of SIG Network",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82966",
-    "pr_number": 82966,
-    "kinds": ["feature"],
-    "sigs": ["network"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "82975": {
-    "commit": "6f5c2507ea53612e411d550d62e2ebc6cd4a9767",
-    "text": "dashboard: disable the dashboard Deployment on non-Linux nodes. This step is required to support Windows worker nodes.",
-    "markdown": "dashboard: disable the dashboard Deployment on non-Linux nodes. This step is required to support Windows worker nodes. ([#82975](https://github.com/kubernetes/kubernetes/pull/82975), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82975",
-    "pr_number": 82975,
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "82996": {
-    "commit": "b00f00931619864a9dba9ffcf0f5d0b24b073aa2",
-    "text": "Fixed the bug that deleted services were processed by EndpointSliceController repeatedly even their cleanup were successful.",
-    "markdown": "Fixed the bug that deleted services were processed by EndpointSliceController repeatedly even their cleanup were successful. ([#82996](https://github.com/kubernetes/kubernetes/pull/82996), [@tnqn](https://github.com/tnqn))",
-    "author": "tnqn",
-    "author_url": "https://github.com/tnqn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82996",
-    "pr_number": 82996,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.17.0"
-  },
-  "83015": {
-    "commit": "9b376e4e8e841c36667873d1ffb3f9b1abfed315",
-    "text": "Bump metrics-server to v0.3.5",
-    "markdown": "Bump metrics-server to v0.3.5 ([#83015](https://github.com/kubernetes/kubernetes/pull/83015), [@olagacek](https://github.com/olagacek))",
-    "author": "olagacek",
-    "author_url": "https://github.com/olagacek",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83015",
-    "pr_number": 83015,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "83027": {
-    "commit": "14d889f6efbee65671e8de7ff02ea02ebda5347c",
-    "text": "Fixes kube-proxy bug accessing self nodeip:port on windows",
-    "markdown": "Fixes kube-proxy bug accessing self nodeip:port on windows ([#83027](https://github.com/kubernetes/kubernetes/pull/83027), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83027",
-    "pr_number": 83027,
-    "areas": ["dependency"],
-    "kinds": ["bug"],
-    "sigs": ["network", "windows"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83030": {
-    "commit": "6dde8daa767d0d51cb3c93a4ff966498c5210a13",
-    "text": "kube-scheduler: a configuration file specified via `--config` is now loaded with strict deserialization, which fails if the config file contains duplicate or unknown fields. This protects against accidentally running with config files that are malformed, mis-indented, or have typos in field names, and getting unexpected behavior.",
-    "markdown": "kube-scheduler: a configuration file specified via `--config` is now loaded with strict deserialization, which fails if the config file contains duplicate or unknown fields. This protects against accidentally running with config files that are malformed, mis-indented, or have typos in field names, and getting unexpected behavior. ([#83030](https://github.com/kubernetes/kubernetes/pull/83030), [@obitech](https://github.com/obitech))\n\n  Courtesy of SIG API Machinery, SIG Cluster Lifecycle, and SIG Scheduling",
-    "author": "obitech",
-    "author_url": "https://github.com/obitech",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83030",
-    "pr_number": 83030,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "scheduling"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83035": {
-    "commit": "7de6f0eb95b6b35ed0426622fc0fc30543ac6c3e",
-    "text": "Improved performance of kube-proxy with EndpointSlice enabled with more efficient sorting.",
-    "markdown": "Improved performance of kube-proxy with EndpointSlice enabled with more efficient sorting. ([#83035](https://github.com/kubernetes/kubernetes/pull/83035), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83035",
-    "pr_number": 83035,
-    "kinds": ["cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.17.0"
-  },
-  "83036": {
-    "commit": "f243640e222913d122a0dee4a47ca4c1609e6123",
-    "text": "Use ipv4 in wincat port forward.",
-    "markdown": "Use ipv4 in wincat port forward. ([#83036](https://github.com/kubernetes/kubernetes/pull/83036), [@liyanhui1228](https://github.com/liyanhui1228))",
-    "author": "liyanhui1228",
-    "author_url": "https://github.com/liyanhui1228",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83036",
-    "pr_number": 83036,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.17.0"
-  },
-  "83042": {
-    "commit": "5e4abfb94613339d585e5d811e4fcec278ff06dc",
-    "text": "Consolidate ScoreWithNormalizePlugin into the ScorePlugin interface",
-    "markdown": "Consolidate ScoreWithNormalizePlugin into the ScorePlugin interface ([#83042](https://github.com/kubernetes/kubernetes/pull/83042), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83042",
-    "pr_number": 83042,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83057": {
-    "commit": "d3593c07de4c6a6ef65a54200b3dd0c8d9228f95",
-    "text": "When using Containerd on Windows, the ``TerminationMessagePath`` file will now be mounted in the Windows Pod.",
-    "markdown": "When using Containerd on Windows, the ``TerminationMessagePath`` file will now be mounted in the Windows Pod. ([#83057](https://github.com/kubernetes/kubernetes/pull/83057), [@claudiubelu](https://github.com/claudiubelu))",
-    "author": "claudiubelu",
-    "author_url": "https://github.com/claudiubelu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83057",
-    "pr_number": 83057,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "windows"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83093": {
-    "commit": "1bfcfa68a9d5e03d9225833d9d15939e66be8593",
-    "text": "The deprecated mondo `kubernetes-test` tarball is no longer built. Users running Kubernetes e2e tests should use the `kubernetes-test-portable` and `kubernetes-test-{OS}-{ARCH}` tarballs instead.",
-    "markdown": "The deprecated mondo `kubernetes-test` tarball is no longer built. Users running Kubernetes e2e tests should use the `kubernetes-test-portable` and `kubernetes-test-{OS}-{ARCH}` tarballs instead. ([#83093](https://github.com/kubernetes/kubernetes/pull/83093), [@ixdy](https://github.com/ixdy))",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-testing/20190118-breaking-apart-the-kubernetes-test-tarball.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "ixdy",
-    "author_url": "https://github.com/ixdy",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83093",
-    "pr_number": 83093,
-    "areas": ["release-eng"],
-    "kinds": ["cleanup"],
-    "sigs": ["release", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83102": {
-    "commit": "77d0ebe39fde51adf3aa78e9ae70cb7acc98182c",
-    "text": "Fix aggressive VM calls for Azure VMSS",
-    "markdown": "Fix aggressive VM calls for Azure VMSS ([#83102](https://github.com/kubernetes/kubernetes/pull/83102), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83102",
-    "pr_number": 83102,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.17.0"
-  },
-  "83104": {
-    "commit": "24424e26d1b2f898b8a26618d0414862f04f2917",
-    "text": "IP validates if a string is a valid IP address",
-    "markdown": "IP validates if a string is a valid IP address ([#83104](https://github.com/kubernetes/kubernetes/pull/83104), [@zouyee](https://github.com/zouyee))",
-    "author": "zouyee",
-    "author_url": "https://github.com/zouyee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83104",
-    "pr_number": 83104,
-    "kinds": ["flake"],
-    "sigs": ["storage"],
-    "release_version": "1.17.0"
-  },
-  "83139": {
-    "commit": "66755bda05b73305c4ed855d1c106b17f0dd3de7",
-    "text": "Update to go 1.12.10",
-    "markdown": "Update to go 1.12.10 ([#83139](https://github.com/kubernetes/kubernetes/pull/83139), [@cblecker](https://github.com/cblecker))",
-    "author": "cblecker",
-    "author_url": "https://github.com/cblecker",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83139",
-    "pr_number": 83139,
-    "areas": ["release-eng", "test"],
-    "kinds": ["bug"],
-    "sigs": ["release", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83184": {
-    "commit": "91b2a7af8745ba610a7749ece553d06495178535",
-    "text": "Fix error where metrics related to dynamic kubelet config isn't registered",
-    "markdown": "Fix error where metrics related to dynamic kubelet config isn't registered ([#83184](https://github.com/kubernetes/kubernetes/pull/83184), [@odinuge](https://github.com/odinuge))",
-    "author": "odinuge",
-    "author_url": "https://github.com/odinuge",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83184",
-    "pr_number": 83184,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.17.0"
-  },
-  "83195": {
-    "commit": "006378fcc1c8db4f7063812b67a59969fc0d09f0",
-    "text": "Promote WatchBookmark feature to GA.\nWith WatchBookmark feature, clients are able to request watch events with BOOKMARK type. Clients should not assume bookmarks are returned at any specific interval, nor may they assume the server will send any BOOKMARK event during a session.",
-    "markdown": "Promote WatchBookmark feature to GA.\n  With WatchBookmark feature, clients are able to request watch events with BOOKMARK type. Clients should not assume bookmarks are returned at any specific interval, nor may they assume the server will send any BOOKMARK event during a session. ([#83195](https://github.com/kubernetes/kubernetes/pull/83195), [@wojtek-t](https://github.com/wojtek-t))",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83195",
-    "pr_number": 83195,
-    "areas": ["apiserver", "code-generation"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "83196": {
-    "commit": "be1574b741cc926a77ebde551b74cdabb6a0be7f",
-    "text": "Use online nodes instead of possible nodes when discovering available NUMA nodes",
-    "markdown": "Use online nodes instead of possible nodes when discovering available NUMA nodes ([#83196](https://github.com/kubernetes/kubernetes/pull/83196), [@zouyee](https://github.com/zouyee))",
-    "author": "zouyee",
-    "author_url": "https://github.com/zouyee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83196",
-    "pr_number": 83196,
-    "areas": ["kubelet"],
-    "kinds": ["flake"],
-    "sigs": ["node"],
-    "release_version": "1.17.0"
-  },
-  "83204": {
-    "commit": "3383d7ca66cc863cc8994b9cb89f6701a7699652",
-    "text": "kubelet: a configuration file specified via `--config` is now loaded with strict deserialization, which fails if the config file contains duplicate or unknown fields. This protects against accidentally running with config files that are malformed, mis-indented, or have typos in field names, and getting unexpected behavior.",
-    "markdown": "kubelet: a configuration file specified via `--config` is now loaded with strict deserialization, which fails if the config file contains duplicate or unknown fields. This protects against accidentally running with config files that are malformed, mis-indented, or have typos in field names, and getting unexpected behavior. ([#83204](https://github.com/kubernetes/kubernetes/pull/83204), [@obitech](https://github.com/obitech))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Node",
-    "author": "obitech",
-    "author_url": "https://github.com/obitech",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83204",
-    "pr_number": 83204,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "node"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83206": {
-    "commit": "30ba9f65480601790e85dcdbfbce5171fe180ca3",
-    "text": "Significant kube-proxy performance improvements when using Endpoint Slices at scale.",
-    "markdown": "Significant kube-proxy performance improvements when using Endpoint Slices at scale. ([#83206](https://github.com/kubernetes/kubernetes/pull/83206), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83206",
-    "pr_number": 83206,
-    "kinds": ["cleanup"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83208": {
-    "commit": "35d68586dbe37ea2d317baabeee86a4db6d1f33f",
-    "text": "Significant kube-proxy performance improvements for non UDP ports.",
-    "markdown": "Significant kube-proxy performance improvements for non UDP ports. ([#83208](https://github.com/kubernetes/kubernetes/pull/83208), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83208",
-    "pr_number": 83208,
-    "areas": ["ipvs"],
-    "kinds": ["cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.17.0"
-  },
-  "83257": {
-    "commit": "0c32aa8910e83075071f8788828d99bcad6e7073",
-    "text": "Adding initial EndpointSlice metrics.",
-    "markdown": "Adding initial EndpointSlice metrics. ([#83257](https://github.com/kubernetes/kubernetes/pull/83257), [@robscott](https://github.com/robscott))\n\n  Courtesy of SIG Apps, and SIG Network",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83257",
-    "pr_number": 83257,
-    "kinds": ["feature"],
-    "sigs": ["apps", "network"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83261": {
-    "commit": "4afcba42bed2bb7c36e5209a90d87343f32a0efa",
-    "text": "Fixes a flaw (CVE-2019-11253) in json/yaml decoding where large or malformed documents could consume excessive server resources. Request bodies for normal API requests (create/delete/update/patch operations of regular resources) are now limited to 3MB.",
-    "markdown": "Fixes a flaw (CVE-2019-11253) in json/yaml decoding where large or malformed documents could consume excessive server resources. Request bodies for normal API requests (create/delete/update/patch operations of regular resources) are now limited to 3MB. ([#83261](https://github.com/kubernetes/kubernetes/pull/83261), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83261",
-    "pr_number": 83261,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli", "cloud-provider", "cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83262": {
-    "commit": "9d59ed80f53a2828343105646f23142a31df8473",
-    "text": "Some scheduler extender API fields are moved from `pkg/scheduler/api` to `pkg/scheduler/apis/extender/v1`.",
-    "markdown": "Some scheduler extender API fields are moved from `pkg/scheduler/api` to `pkg/scheduler/apis/extender/v1`. ([#83262](https://github.com/kubernetes/kubernetes/pull/83262), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83262",
-    "pr_number": 83262,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83276": {
-    "commit": "6564b34f9e274f62ce38ff11b54393c1f3e4d7fd",
-    "text": "`kubeadm.k8s.io/v1beta1` has been deprecated, you should update your config to use newer non-deprecated API versions.",
-    "markdown": "`kubeadm.k8s.io/v1beta1` has been deprecated, you should update your config to use newer non-deprecated API versions. ([#83276](https://github.com/kubernetes/kubernetes/pull/83276), [@Klaven](https://github.com/Klaven))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "Klaven",
-    "author_url": "https://github.com/Klaven",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83276",
-    "pr_number": 83276,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "action_required": true,
-    "release_version": "1.17.0"
-  },
-  "83283": {
-    "commit": "7cc611b7e01191f262150def59cd39ad7eaaffed",
-    "text": "ETCD version monitor metrics are now marked as with the ALPHA stability level.",
-    "markdown": "ETCD version monitor metrics are now marked as with the ALPHA stability level. ([#83283](https://github.com/kubernetes/kubernetes/pull/83283), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/13a778c6a4a84dbde9e691e8dcf930a6eaa7ca51/keps/sig-instrumentation/20190605-metrics-stability-migration.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83283",
-    "pr_number": 83283,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83333": {
-    "commit": "acd9141e76d3453c08700f22b24bdc320718d5dc",
-    "text": "Fixes a goroutine leak in kube-apiserver when a request times out.",
-    "markdown": "Fixes a goroutine leak in kube-apiserver when a request times out. ([#83333](https://github.com/kubernetes/kubernetes/pull/83333), [@lavalamp](https://github.com/lavalamp))",
-    "author": "lavalamp",
-    "author_url": "https://github.com/lavalamp",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83333",
-    "pr_number": 83333,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "83386": {
-    "commit": "8a123346838d4b96b68835aebf45a4334867113c",
-    "text": "Remove MaxPriority in the scheduler API, please use MaxNodeScore or MaxExtenderPriority instead.",
-    "markdown": "Remove MaxPriority in the scheduler API, please use MaxNodeScore or MaxExtenderPriority instead. ([#83386](https://github.com/kubernetes/kubernetes/pull/83386), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83386",
-    "pr_number": 83386,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83394": {
-    "commit": "5e30d32d2cb5cab6f23269a219b59d69a2b969eb",
-    "text": "Existing PVs are converted to use volume topology if migration is enabled.",
-    "markdown": "Existing PVs are converted to use volume topology if migration is enabled. ([#83394](https://github.com/kubernetes/kubernetes/pull/83394), [@bertinatto](https://github.com/bertinatto))\n\n  Courtesy of SIG Apps, SIG Scheduling, and SIG Storage",
-    "author": "bertinatto",
-    "author_url": "https://github.com/bertinatto",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83394",
-    "pr_number": 83394,
-    "kinds": ["feature"],
-    "sigs": ["apps", "scheduling", "storage"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83396": {
-    "commit": "56f60d1bcc81270fd9239ddb6b8ed83b1beb77a6",
-    "text": "Bump version of event-exporter to 0.3.1, to switch it to protobuf.",
-    "markdown": "Bump version of event-exporter to 0.3.1, to switch it to protobuf. ([#83396](https://github.com/kubernetes/kubernetes/pull/83396), [@loburm](https://github.com/loburm))\n\n  Courtesy of SIG Instrumentation, and SIG Scalability",
-    "author": "loburm",
-    "author_url": "https://github.com/loburm",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83396",
-    "pr_number": 83396,
-    "kinds": ["feature"],
-    "sigs": ["instrumentation", "scalability"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83419": {
-    "commit": "cb19b56831d54d1d31249949318ef0b07bf00df9",
-    "text": "An end-user may choose to request logs without confirming the identity of the backing kubelet.  This feature can be disabled by setting the `AllowInsecureBackendProxy` feature-gate to false.",
-    "markdown": "An end-user may choose to request logs without confirming the identity of the backing kubelet.  This feature can be disabled by setting the `AllowInsecureBackendProxy` feature-gate to false. ([#83419](https://github.com/kubernetes/kubernetes/pull/83419), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83419",
-    "pr_number": 83419,
-    "areas": ["kubelet", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["apps", "node", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83427": {
-    "commit": "aa25739da416b68e966bef3c15e4f5281ebd1e85",
-    "text": "Adds a metric apiserver_request_error_total to kube-apiserver. This metric tallies the number of request_errors encountered by verb, group, version, resource, subresource, scope, component, and code.",
-    "markdown": "Adds a metric apiserver_request_error_total to kube-apiserver. This metric tallies the number of request_errors encountered by verb, group, version, resource, subresource, scope, component, and code. ([#83427](https://github.com/kubernetes/kubernetes/pull/83427), [@logicalhan](https://github.com/logicalhan))",
-    "author": "logicalhan",
-    "author_url": "https://github.com/logicalhan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83427",
-    "pr_number": 83427,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83430": {
-    "commit": "eba349b855c0ac70d4a954127cb5ed358cc0a8ab",
-    "text": "Rename PluginContext to CycleState in the scheduling framework",
-    "markdown": "Rename PluginContext to CycleState in the scheduling framework ([#83430](https://github.com/kubernetes/kubernetes/pull/83430), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83430",
-    "pr_number": 83430,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83442": {
-    "commit": "a8e819746d0fbcbcafc715081177caa9b580d016",
-    "text": "Remove prometheus cluster monitoring addon from kube-up",
-    "markdown": "Remove prometheus cluster monitoring addon from kube-up ([#83442](https://github.com/kubernetes/kubernetes/pull/83442), [@serathius](https://github.com/serathius))",
-    "author": "serathius",
-    "author_url": "https://github.com/serathius",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83442",
-    "pr_number": 83442,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83451": {
-    "commit": "8098bae92114a85134f30aa0a6d7bb72c37412ae",
-    "text": "Fixed cleanup of raw block devices after kubelet restart.",
-    "markdown": "Fixed cleanup of raw block devices after kubelet restart. ([#83451](https://github.com/kubernetes/kubernetes/pull/83451), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83451",
-    "pr_number": 83451,
-    "areas": ["kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["node", "storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83454": {
-    "commit": "7c540ecdac946edfefe03bba50d1dd08ec4b62d5",
-    "text": "hyperkube will now be available in a new github repository and will not be included in the kubernetes release from 1.17 onwards",
-    "markdown": "hyperkube will now be available in a new github repository and will not be included in the kubernetes release from 1.17 onwards ([#83454](https://github.com/kubernetes/kubernetes/pull/83454), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83454",
-    "pr_number": 83454,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "release"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83474": {
-    "commit": "1d1385af915098abd8bd23ab397ee0aeaa244f1e",
-    "text": "CSI Topology feature is GA. The CSINodeInfo feature gate is deprecated and will be removed in a future release. The storage.k8s.io/v1beta1 CSINode object is deprecated and will be removed in a future release.",
-    "markdown": "CSI Topology feature is GA. The CSINodeInfo feature gate is deprecated and will be removed in a future release. The storage.k8s.io/v1beta1 CSINode object is deprecated and will be removed in a future release. ([#83474](https://github.com/kubernetes/kubernetes/pull/83474), [@msau42](https://github.com/msau42))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG CLI, SIG Storage, and SIG Testing",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/ac738ac78ec2e425c54354119526bac916baeb7e/keps/sig-storage/20190124-csi-topology.md",
-        "type": "KEP"
-      },
-      {
-        "description": "[Usage]",
-        "url": "https://kubernetes.io/docs/concepts/storage/storage-classes/#volume-binding-mode",
-        "type": "official"
-      },
-      {
-        "description": "[Other doc]",
-        "url": "https://kubernetes-csi.github.io/docs/topology.html",
-        "type": "external"
-      }
-    ],
-    "author": "msau42",
-    "author_url": "https://github.com/msau42",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83474",
-    "pr_number": 83474,
-    "areas": ["apiserver", "kubectl", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "auth", "cli", "storage", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83475": {
-    "commit": "5a4c6c2b34a011ca1801354395538df5b8e8cd66",
-    "text": "Kubeadm: add support for 127.0.0.1 as advertise address. kubeadm will automatically replace this value with matching global unicast IP address on the loopback interface.",
-    "markdown": "Kubeadm: add support for 127.0.0.1 as advertise address. kubeadm will automatically replace this value with matching global unicast IP address on the loopback interface. ([#83475](https://github.com/kubernetes/kubernetes/pull/83475), [@fabriziopandini](https://github.com/fabriziopandini))\n\n  Courtesy of SIG API Machinery, and SIG Cluster Lifecycle",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83475",
-    "pr_number": 83475,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83492": {
-    "commit": "017842d49deca68dab3a26b76d5dfd5df15a8113",
-    "text": "The topology manager aligns resources for pods of all QoS classes with respect to NUMA locality, not just Guaranteed QoS pods.",
-    "markdown": "The topology manager aligns resources for pods of all QoS classes with respect to NUMA locality, not just Guaranteed QoS pods. ([#83492](https://github.com/kubernetes/kubernetes/pull/83492), [@ConnorDoyle](https://github.com/ConnorDoyle))\n\n  Courtesy of SIG Node",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/c1e2ceed191fa3b1d7bc0a1e3ae6c472083ad09b/keps/sig-node/0035-20190130-topology-manager.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "ConnorDoyle",
-    "author_url": "https://github.com/ConnorDoyle",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83492",
-    "pr_number": 83492,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83508": {
-    "commit": "a381f7cb3efa64395661c73961c07b02594acf64",
-    "text": "Scheduler now reports metrics on cache size including nodes, pods, and assumed pods",
-    "markdown": "Scheduler now reports metrics on cache size including nodes, pods, and assumed pods ([#83508](https://github.com/kubernetes/kubernetes/pull/83508), [@damemi](https://github.com/damemi))\n\n  Courtesy of SIG Instrumentation, and SIG Scheduling",
-    "author": "damemi",
-    "author_url": "https://github.com/damemi",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83508",
-    "pr_number": 83508,
-    "kinds": ["feature"],
-    "sigs": ["instrumentation", "scheduling"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83517": {
-    "commit": "569a45fb02fc3065c7051cc7bc91fabf0e24b09c",
-    "text": "Utilize diagnostics tool to dump GKE windows test logs",
-    "markdown": "Utilize diagnostics tool to dump GKE windows test logs ([#83517](https://github.com/kubernetes/kubernetes/pull/83517), [@YangLu1031](https://github.com/YangLu1031))",
-    "author": "YangLu1031",
-    "author_url": "https://github.com/YangLu1031",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83517",
-    "pr_number": 83517,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "83520": {
-    "commit": "8ed2f4775a0d2b13c4be790cdfc1f34bc8b6522b",
-    "text": "When the go-client reflector relists, the ResourceVersion list option is set to the reflector's latest synced resource version to ensure the reflector does not \"go back in time\" and reprocess events older than it has already processed. If the the server responds with an HTTP 410 (Gone) status code response, the relist falls back to using resourceVersion=\"\".",
-    "markdown": "When the go-client reflector relists, the ResourceVersion list option is set to the reflector's latest synced resource version to ensure the reflector does not \"go back in time\" and reprocess events older than it has already processed. If the the server responds with an HTTP 410 (Gone) status code response, the relist falls back to using resourceVersion=\"\". ([#83520](https://github.com/kubernetes/kubernetes/pull/83520), [@jpbetz](https://github.com/jpbetz))",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83520",
-    "pr_number": 83520,
-    "areas": ["apiserver", "dependency", "kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "node"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83522": {
-    "commit": "1e5a9f3ff4eef0b190c4300037607097b8027439",
-    "text": "Expand scheduler priority functions and scheduling framework plugins' node score range to [0, 100]. action required. Note: this change is internal and does not affect extender and RequestedToCapacityRatio custom priority, which are still expected to provide a [0, 10] range.",
-    "markdown": "Expand scheduler priority functions and scheduling framework plugins' node score range to [0, 100]. action required. Note: this change is internal and does not affect extender and RequestedToCapacityRatio custom priority, which are still expected to provide a [0, 10] range. ([#83522](https://github.com/kubernetes/kubernetes/pull/83522), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Scheduling",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83522",
-    "pr_number": 83522,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "action_required": true,
-    "release_version": "1.17.0"
-  },
-  "83528": {
-    "commit": "c8d4d626d8e9baaee885f85cd2476395e597ec14",
-    "text": "kubeadm: fix wrong default value for the \"upgrade node --certificate-renewal\" flag.",
-    "markdown": "kubeadm: fix wrong default value for the \"upgrade node --certificate-renewal\" flag. ([#83528](https://github.com/kubernetes/kubernetes/pull/83528), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83528",
-    "pr_number": 83528,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "83535": {
-    "commit": "0c3424f53c9b6fe7e880c30e13c1d4aa206027f1",
-    "text": "Add a metric to track number of scheduler binding and prioritizing goroutines",
-    "markdown": "Add a metric to track number of scheduler binding and prioritizing goroutines ([#83535](https://github.com/kubernetes/kubernetes/pull/83535), [@wgliang](https://github.com/wgliang))",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83535",
-    "pr_number": 83535,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.17.0"
-  },
-  "83539": {
-    "commit": "d38f340e4ff6214b3365b05e744b965d0d32bdcf",
-    "text": "Add more tracing steps in generic_scheduler",
-    "markdown": "Add more tracing steps in generic_scheduler ([#83539](https://github.com/kubernetes/kubernetes/pull/83539), [@wgliang](https://github.com/wgliang))",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83539",
-    "pr_number": 83539,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.17.0"
-  },
-  "83547": {
-    "commit": "70d453d32df254ef0c3420804c6487a2ba55a5e0",
-    "text": "The `--certificate-authority` flag now correctly overrides existing skip TLS or CA data settings in the kubeconfig file",
-    "markdown": "The `--certificate-authority` flag now correctly overrides existing skip TLS or CA data settings in the kubeconfig file ([#83547](https://github.com/kubernetes/kubernetes/pull/83547), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83547",
-    "pr_number": 83547,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83551": {
-    "commit": "3db6d3abcf11a40029fa6c8b5096535dfc422368",
-    "text": "external facing APIs in pluginregistration and deviceplugin packages are now available under k8s.io/kubelet/pkg/apis/",
-    "markdown": "external facing APIs in pluginregistration and deviceplugin packages are now available under k8s.io/kubelet/pkg/apis/ ([#83551](https://github.com/kubernetes/kubernetes/pull/83551), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83551",
-    "pr_number": 83551,
-    "areas": ["code-organization", "dependency", "kubelet", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["node", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83552": {
-    "commit": "c42eceb1365cfee18ebf4e09083ce112abc95b3a",
-    "text": "Commands like `kubectl apply` now return errors if schema-invalid annotations are specified, rather than silently dropping the entire annotations section.",
-    "markdown": "Commands like `kubectl apply` now return errors if schema-invalid annotations are specified, rather than silently dropping the entire annotations section. ([#83552](https://github.com/kubernetes/kubernetes/pull/83552), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83552",
-    "pr_number": 83552,
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.17.0"
-  },
-  "83567": {
-    "commit": "1fcd46c250ea0cfe4c84bc5db63d83534101fb1d",
-    "text": "Fixed attachment of AWS volumes that have just been detached.",
-    "markdown": "Fixed attachment of AWS volumes that have just been detached. ([#83567](https://github.com/kubernetes/kubernetes/pull/83567), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83567",
-    "pr_number": 83567,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83568": {
-    "commit": "c580a12c8ea0797c25061a3ff170dcc00e05be79",
-    "text": "Node-specific volume limits has graduated to GA.",
-    "markdown": "Node-specific volume limits has graduated to GA. ([#83568](https://github.com/kubernetes/kubernetes/pull/83568), [@bertinatto](https://github.com/bertinatto))\n\n  Courtesy of SIG Auth, SIG Node, SIG Scheduling, SIG Storage, and SIG Testing",
-    "author": "bertinatto",
-    "author_url": "https://github.com/bertinatto",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83568",
-    "pr_number": 83568,
-    "areas": ["dependency", "kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["auth", "node", "scheduling", "storage", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83569": {
-    "commit": "1552ba6b0021440660e13316062ebbe14075b676",
-    "text": "Add latency and request count metrics for scheduler framework.",
-    "markdown": "Add latency and request count metrics for scheduler framework. ([#83569](https://github.com/kubernetes/kubernetes/pull/83569), [@liu-cong](https://github.com/liu-cong))\n\n  Courtesy of SIG Scheduling",
-    "author": "liu-cong",
-    "author_url": "https://github.com/liu-cong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83569",
-    "pr_number": 83569,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83576": {
-    "commit": "b0c26e5a5af611bf7577a80329889341f21515cf",
-    "text": "update github.com/vishvananda/netlink to v1.0.0",
-    "markdown": "update github.com/vishvananda/netlink to v1.0.0 ([#83576](https://github.com/kubernetes/kubernetes/pull/83576), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83576",
-    "pr_number": 83576,
-    "areas": ["dependency"],
-    "kinds": ["cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.17.0"
-  },
-  "83577": {
-    "commit": "f7091992c04e2d4f16f6b462d4a711e5b9eb0f22",
-    "text": "Add incoming pods metrics to scheduler queue.",
-    "markdown": "Add incoming pods metrics to scheduler queue. ([#83577](https://github.com/kubernetes/kubernetes/pull/83577), [@liu-cong](https://github.com/liu-cong))\n\n  Courtesy of SIG Scheduling",
-    "author": "liu-cong",
-    "author_url": "https://github.com/liu-cong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83577",
-    "pr_number": 83577,
-    "kinds": ["cleanup", "feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83578": {
-    "commit": "2da4678a802696efa7e42a5902216c14470fce7a",
-    "text": "Scheduler Policy API has a new recommended apiVersion \"apiVersion: kubescheduler.config.k8s.io/v1\" which is consistent with the scheduler API group \"kubescheduler.config.k8s.io\". It holds the same API as the old apiVersion \"apiVersion: v1\".",
-    "markdown": "Scheduler Policy API has a new recommended apiVersion \"apiVersion: kubescheduler.config.k8s.io/v1\" which is consistent with the scheduler API group \"kubescheduler.config.k8s.io\". It holds the same API as the old apiVersion \"apiVersion: v1\". ([#83578](https://github.com/kubernetes/kubernetes/pull/83578), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83578",
-    "pr_number": 83578,
-    "areas": ["dependency", "test"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83579": {
-    "commit": "0f1a8054f986d205a0202bf9b378c71e7ecc1710",
-    "text": "client-ca bundles for the all generic-apiserver based servers will dynamically reload from disk on content changes",
-    "markdown": "client-ca bundles for the all generic-apiserver based servers will dynamically reload from disk on content changes ([#83579](https://github.com/kubernetes/kubernetes/pull/83579), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83579",
-    "pr_number": 83579,
-    "areas": ["apiserver", "dependency", "kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth", "node", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83591": {
-    "commit": "18cef954a1452dc0acda14b43a2d61e620fba53c",
-    "text": "Add data cache flushing during unmount device for GCE-PD driver in Windows Server.",
-    "markdown": "Add data cache flushing during unmount device for GCE-PD driver in Windows Server. ([#83591](https://github.com/kubernetes/kubernetes/pull/83591), [@jingxu97](https://github.com/jingxu97))",
-    "author": "jingxu97",
-    "author_url": "https://github.com/jingxu97",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83591",
-    "pr_number": 83591,
-    "kinds": ["bug"],
-    "sigs": ["storage", "windows"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83592": {
-    "commit": "73b2c82b2864eeac5b81eb01e06c5726a11176cf",
-    "text": "A new kubelet command line option, --reserved-cpus, is introduced to explicitly define the the CPU list that will be reserved for system. For example, if --reserved-cpus=0,1,2,3 is specified, then cpu 0,1,2,3 will be reserved for the system.  On a system with 24 CPUs, the user may specify isolcpus=4-23 for the kernel option and use CPU 4-23 for the user containers.",
-    "markdown": "A new kubelet command line option, --reserved-cpus, is introduced to explicitly define the the CPU list that will be reserved for system. For example, if --reserved-cpus=0,1,2,3 is specified, then cpu 0,1,2,3 will be reserved for the system.  On a system with 24 CPUs, the user may specify isolcpus=4-23 for the kernel option and use CPU 4-23 for the user containers. ([#83592](https://github.com/kubernetes/kubernetes/pull/83592), [@jianzzha](https://github.com/jianzzha))\n\n  Courtesy of SIG API Machinery, SIG Cluster Lifecycle, and SIG Node",
-    "author": "jianzzha",
-    "author_url": "https://github.com/jianzzha",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83592",
-    "pr_number": 83592,
-    "areas": ["code-generation", "kubeadm", "kubelet"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "node"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83596": {
-    "commit": "6d97d36fbb5a52e9e7520fb326283a53ba1570a4",
-    "text": "Bumps the minimum version of Go required for building Kubernetes to 1.12.4.",
-    "markdown": "Bumps the minimum version of Go required for building Kubernetes to 1.12.4. ([#83596](https://github.com/kubernetes/kubernetes/pull/83596), [@jktomer](https://github.com/jktomer))",
-    "author": "jktomer",
-    "author_url": "https://github.com/jktomer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83596",
-    "pr_number": 83596,
-    "kinds": ["cleanup"],
-    "sigs": ["release"],
-    "release_version": "1.17.0"
-  },
-  "83599": {
-    "commit": "d8a3fbd27590ff4b56252dc4b2d0b54323a98b98",
-    "text": "kube-proxy iptables probabilities are now more granular and will result in better distribution beyond 319 endpoints.",
-    "markdown": "kube-proxy iptables probabilities are now more granular and will result in better distribution beyond 319 endpoints. ([#83599](https://github.com/kubernetes/kubernetes/pull/83599), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83599",
-    "pr_number": 83599,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.17.0"
-  },
-  "83603": {
-    "commit": "54f089fa08704f3fec01ae429452409acfbd1293",
-    "text": "Change `pod_preemption_victims` metric from Gauge to Histogram.",
-    "markdown": "Change `pod_preemption_victims` metric from Gauge to Histogram. ([#83603](https://github.com/kubernetes/kubernetes/pull/83603), [@Tabrizian](https://github.com/Tabrizian))\n\n  Courtesy of SIG Scheduling",
-    "author": "Tabrizian",
-    "author_url": "https://github.com/Tabrizian",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83603",
-    "pr_number": 83603,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83619": {
-    "commit": "042b17ef197afb34f8e827d65f2658dcc1d63923",
-    "text": "Scheduler ComponentConfig fields are now pointers",
-    "markdown": "Scheduler ComponentConfig fields are now pointers ([#83619](https://github.com/kubernetes/kubernetes/pull/83619), [@damemi](https://github.com/damemi))",
-    "author": "damemi",
-    "author_url": "https://github.com/damemi",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83619",
-    "pr_number": 83619,
-    "areas": ["test"],
-    "kinds": ["api-change"],
-    "sigs": ["scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83643": {
-    "commit": "6992d1386c481437acbfc2a7edcf4ae3575cc85c",
-    "text": "Authentication token cache size is increased (from 4k to 32k) to support clusters with many nodes or many namespaces with active service accounts.",
-    "markdown": "Authentication token cache size is increased (from 4k to 32k) to support clusters with many nodes or many namespaces with active service accounts. ([#83643](https://github.com/kubernetes/kubernetes/pull/83643), [@lavalamp](https://github.com/lavalamp))",
-    "author": "lavalamp",
-    "author_url": "https://github.com/lavalamp",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83643",
-    "pr_number": 83643,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83644": {
-    "commit": "bb6166419ac399d6b0007eb5657c6b94f182d9b0",
-    "text": "The userspace mode of kube-proxy no longer confusingly logs messages about deleting endpoints that it is actually adding.",
-    "markdown": "The userspace mode of kube-proxy no longer confusingly logs messages about deleting endpoints that it is actually adding. ([#83644](https://github.com/kubernetes/kubernetes/pull/83644), [@danwinship](https://github.com/danwinship))",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83644",
-    "pr_number": 83644,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.17.0"
-  },
-  "83650": {
-    "commit": "a78340410e519f4e9870e1b29316572e63e59a91",
-    "text": "[migration phase 1] PodFitsResources as framework plugin",
-    "markdown": "[migration phase 1] PodFitsResources as framework plugin ([#83650](https://github.com/kubernetes/kubernetes/pull/83650), [@wgliang](https://github.com/wgliang))",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83650",
-    "pr_number": 83650,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.17.0"
-  },
-  "83659": {
-    "commit": "5a427e87043e473e2768556de4560d51bb385e6a",
-    "text": "[migration phase 1] PodFitsHostPorts as filter plugin",
-    "markdown": "[migration phase 1] PodFitsHostPorts as filter plugin ([#83659](https://github.com/kubernetes/kubernetes/pull/83659), [@wgliang](https://github.com/wgliang))",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83659",
-    "pr_number": 83659,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.17.0"
-  },
-  "83660": {
-    "commit": "becc1b945b54d982fc31457ac550923895192cbd",
-    "text": "[migration phase 1] PodMatchNodeSelector/NodAffinity as filter plugin",
-    "markdown": "[migration phase 1] PodMatchNodeSelector/NodAffinity as filter plugin ([#83660](https://github.com/kubernetes/kubernetes/pull/83660), [@wgliang](https://github.com/wgliang))",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83660",
-    "pr_number": 83660,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.17.0"
-  },
-  "83662": {
-    "commit": "afa394bd493304a8d940e2b74e5e71c46e972fcd",
-    "text": "[migration phase 1] PodFitsHost as filter plugin",
-    "markdown": "[migration phase 1] PodFitsHost as filter plugin ([#83662](https://github.com/kubernetes/kubernetes/pull/83662), [@wgliang](https://github.com/wgliang))",
-    "author": "wgliang",
-    "author_url": "https://github.com/wgliang",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83662",
-    "pr_number": 83662,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.17.0"
-  },
-  "83663": {
-    "commit": "c980000176b527aec3cf925d3be2d8f8ca546e27",
-    "text": "Expose SharedInformerFactory in the framework handle",
-    "markdown": "Expose SharedInformerFactory in the framework handle ([#83663](https://github.com/kubernetes/kubernetes/pull/83663), [@draveness](https://github.com/draveness))\n\n  Courtesy of SIG Apps, SIG Scheduling, and SIG Testing",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83663",
-    "pr_number": 83663,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["apps", "scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83674": {
-    "commit": "c41942b81dc2561deb36239828d5f99692547e02",
-    "text": "Add per-pod scheduling metrics across 1 or more schedule attempts.",
-    "markdown": "Add per-pod scheduling metrics across 1 or more schedule attempts. ([#83674](https://github.com/kubernetes/kubernetes/pull/83674), [@liu-cong](https://github.com/liu-cong))\n\n  Courtesy of SIG Scheduling",
-    "author": "liu-cong",
-    "author_url": "https://github.com/liu-cong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83674",
-    "pr_number": 83674,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83677": {
-    "commit": "bbe71214a6800ad7e3ee03bf564e4afcfb40de77",
-    "text": "Updated kube-proxy ipvs README with correct grep argument to list loaded ipvs modules",
-    "markdown": "Updated kube-proxy ipvs README with correct grep argument to list loaded ipvs modules ([#83677](https://github.com/kubernetes/kubernetes/pull/83677), [@pete911](https://github.com/pete911))",
-    "author": "pete911",
-    "author_url": "https://github.com/pete911",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83677",
-    "pr_number": 83677,
-    "areas": ["ipvs"],
-    "kinds": ["documentation"],
-    "sigs": ["network"],
-    "release_version": "1.17.0"
-  },
-  "83685": {
-    "commit": "11ab52a74a4651ba7179123a478f9a9a8d503285",
-    "text": "azure: Add allow unsafe read from cache",
-    "markdown": "azure: Add allow unsafe read from cache ([#83685](https://github.com/kubernetes/kubernetes/pull/83685), [@aramase](https://github.com/aramase))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "aramase",
-    "author_url": "https://github.com/aramase",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83685",
-    "pr_number": 83685,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug", "feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83692": {
-    "commit": "486e2380bd2847f1af7f3c1d0c7b01e7e688415e",
-    "text": "kube-scheduler now fallbacks to emitting events using core/v1 Events when events.k8s.io/v1beta1 is disabled.",
-    "markdown": "kube-scheduler now fallbacks to emitting events using core/v1 Events when events.k8s.io/v1beta1 is disabled. ([#83692](https://github.com/kubernetes/kubernetes/pull/83692), [@yastij](https://github.com/yastij))",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83692",
-    "pr_number": 83692,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "apps", "scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83697": {
-    "commit": "4561b67971213cad640ccb26d3c5830fff864621",
-    "text": "Fixed a bug in the single-numa-node policy of the TopologyManager.  Previously, pods that only requested CPU resources and did not request any third-party devices would fail to launch with a TopologyAffinity error. Now they will launch successfully.",
-    "markdown": "Fixed a bug in the single-numa-node policy of the TopologyManager.  Previously, pods that only requested CPU resources and did not request any third-party devices would fail to launch with a TopologyAffinity error. Now they will launch successfully. ([#83697](https://github.com/kubernetes/kubernetes/pull/83697), [@klueska](https://github.com/klueska))",
-    "author": "klueska",
-    "author_url": "https://github.com/klueska",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83697",
-    "pr_number": 83697,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.17.0"
-  },
-  "83756": {
-    "commit": "7e53c9d80808e44ae0b320f01fe46a03eda2ccf6",
-    "text": "Refactor scheduler's framework permit API.",
-    "markdown": "Refactor scheduler's framework permit API. ([#83756](https://github.com/kubernetes/kubernetes/pull/83756), [@hex108](https://github.com/hex108))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "hex108",
-    "author_url": "https://github.com/hex108",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83756",
-    "pr_number": 83756,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83777": {
-    "commit": "002dbf6a4c67fb73ff7a6adec88ae82f03a82cbc",
-    "text": "Fixed a bug in the single-numa-policy of the TopologyManager. Previously, best-effort pods would result in a terminated state with a TopologyAffinity error. Now they will run as expected.",
-    "markdown": "Fixed a bug in the single-numa-policy of the TopologyManager. Previously, best-effort pods would result in a terminated state with a TopologyAffinity error. Now they will run as expected. ([#83777](https://github.com/kubernetes/kubernetes/pull/83777), [@lmdaly](https://github.com/lmdaly))",
-    "author": "lmdaly",
-    "author_url": "https://github.com/lmdaly",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83777",
-    "pr_number": 83777,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node"],
-    "release_version": "1.17.0"
-  },
-  "83787": {
-    "commit": "f86b0814a5f78b8813a6c11835285dce421f8e1e",
-    "text": "Fixed panic when accessing CustomResources of a CRD with x-kubernetes-int-or-string.",
-    "markdown": "Fixed panic when accessing CustomResources of a CRD with x-kubernetes-int-or-string. ([#83787](https://github.com/kubernetes/kubernetes/pull/83787), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83787",
-    "pr_number": 83787,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "83801": {
-    "commit": "5e2650883af75cfb09eb4a65b4fb0e7d867a104f",
-    "text": "Upgrade to etcd client 3.3.17 to fix bug where etcd client does not parse IPv6 addresses correctly when members are joining, and to fix bug where failover on multi-member etcd cluster fails certificate check on DNS mismatch",
-    "markdown": "Upgrade to etcd client 3.3.17 to fix bug where etcd client does not parse IPv6 addresses correctly when members are joining, and to fix bug where failover on multi-member etcd cluster fails certificate check on DNS mismatch ([#83801](https://github.com/kubernetes/kubernetes/pull/83801), [@jpbetz](https://github.com/jpbetz))",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83801",
-    "pr_number": 83801,
-    "areas": ["apiserver", "cloudprovider", "dependency", "etcd"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83804": {
-    "commit": "cb3b715de2695044e998bce5377800a7ea6558c1",
-    "text": "Upgrade default etcd server version to 3.3.17",
-    "markdown": "Upgrade default etcd server version to 3.3.17 ([#83804](https://github.com/kubernetes/kubernetes/pull/83804), [@jpbetz](https://github.com/jpbetz))",
-    "author": "jpbetz",
-    "author_url": "https://github.com/jpbetz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83804",
-    "pr_number": 83804,
-    "areas": ["etcd", "kubeadm", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83815": {
-    "commit": "7d1f9b440ef5ce93c43bd07fe4a760c7325dffb6",
-    "text": "Added appProtocol field to EndpointSlice Port",
-    "markdown": "Added appProtocol field to EndpointSlice Port ([#83815](https://github.com/kubernetes/kubernetes/pull/83815), [@howardjohn](https://github.com/howardjohn))",
-    "author": "howardjohn",
-    "author_url": "https://github.com/howardjohn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83815",
-    "pr_number": 83815,
-    "areas": ["dependency"],
-    "kinds": ["api-change"],
-    "sigs": ["apps", "cluster-lifecycle", "instrumentation", "network"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83822": {
-    "commit": "1732b435b33e0d7061f928f531ca47557045e589",
-    "text": "set config.BindAddress to IPv4 address \"127.0.0.1\" if not specified",
-    "markdown": "set config.BindAddress to IPv4 address \"127.0.0.1\" if not specified ([#83822](https://github.com/kubernetes/kubernetes/pull/83822), [@zouyee](https://github.com/zouyee))",
-    "author": "zouyee",
-    "author_url": "https://github.com/zouyee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83822",
-    "pr_number": 83822,
-    "areas": ["ipvs"],
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.17.0"
-  },
-  "83825": {
-    "commit": "2afb403fa1a0b7f3b916bf6fad35aeb1b9a22d25",
-    "text": "Gives the right error message when using `kubectl delete` a wrong resource.",
-    "markdown": "Gives the right error message when using `kubectl delete` a wrong resource. ([#83825](https://github.com/kubernetes/kubernetes/pull/83825), [@zhouya0](https://github.com/zhouya0))",
-    "author": "zhouya0",
-    "author_url": "https://github.com/zhouya0",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83825",
-    "pr_number": 83825,
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.17.0"
-  },
-  "83832": {
-    "commit": "dc8880908f1799f3a42739817c5f4ac10383facb",
-    "text": "deprecate cleanup-ipvs flag",
-    "markdown": "deprecate cleanup-ipvs flag ([#83832](https://github.com/kubernetes/kubernetes/pull/83832), [@gongguan](https://github.com/gongguan))",
-    "author": "gongguan",
-    "author_url": "https://github.com/gongguan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83832",
-    "pr_number": 83832,
-    "areas": ["ipvs"],
-    "kinds": ["cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.17.0"
-  },
-  "83836": {
-    "commit": "e45e6e5f2008faba76f03072f6af48ada5e8a50f",
-    "text": "Deprecated metric `rest_client_request_latency_seconds` has been turned off.",
-    "markdown": "Deprecated metric `rest_client_request_latency_seconds` has been turned off. ([#83836](https://github.com/kubernetes/kubernetes/pull/83836), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG API Machinery, SIG Cluster Lifecycle, and SIG Instrumentation",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/da4b7050ccae7a947e4d60f94ab28513e513a458/keps/sig-instrumentation/20181106-kubernetes-metrics-overhaul.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83836",
-    "pr_number": 83836,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "instrumentation"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83837": {
-    "commit": "beaf3a2f04f2882b9e1de99e4b4f2f06995e3d82",
-    "text": "Following metrics have been turned off:\n- apiserver_request_count\n- apiserver_request_latencies\n- apiserver_request_latencies_summary\n- apiserver_dropped_requests\n- etcd_request_latencies_summary\n- apiserver_storage_transformation_latencies_microseconds\n- apiserver_storage_data_key_generation_latencies_microseconds\n- apiserver_storage_transformation_failures_total",
-    "markdown": "Following metrics have been turned off:\n  - apiserver_request_count\n  - apiserver_request_latencies\n  - apiserver_request_latencies_summary\n  - apiserver_dropped_requests\n  - etcd_request_latencies_summary\n  - apiserver_storage_transformation_latencies_microseconds\n  - apiserver_storage_data_key_generation_latencies_microseconds\n  - apiserver_storage_transformation_failures_total ([#83837](https://github.com/kubernetes/kubernetes/pull/83837), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG API Machinery, and SIG Testing",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/da4b7050ccae7a947e4d60f94ab28513e513a458/keps/sig-instrumentation/20181106-kubernetes-metrics-overhaul.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83837",
-    "pr_number": 83837,
-    "areas": ["apiserver", "e2e-test-framework", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83838": {
-    "commit": "7a5929da02f9fbdb9d63b5335f159462cc553020",
-    "text": "Following metrics have been turned off:\n- scheduler_scheduling_latency_seconds\n- scheduler_e2e_scheduling_latency_microseconds\n- scheduler_scheduling_algorithm_latency_microseconds\n- scheduler_scheduling_algorithm_predicate_evaluation\n- scheduler_scheduling_algorithm_priority_evaluation\n- scheduler_scheduling_algorithm_preemption_evaluation\n- scheduler_scheduling_binding_latency_microseconds",
-    "markdown": "Following metrics have been turned off:\n  - scheduler_scheduling_latency_seconds\n  - scheduler_e2e_scheduling_latency_microseconds\n  - scheduler_scheduling_algorithm_latency_microseconds\n  - scheduler_scheduling_algorithm_predicate_evaluation\n  - scheduler_scheduling_algorithm_priority_evaluation\n  - scheduler_scheduling_algorithm_preemption_evaluation\n  - scheduler_scheduling_binding_latency_microseconds ([#83838](https://github.com/kubernetes/kubernetes/pull/83838), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Scheduling",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/da4b7050ccae7a947e4d60f94ab28513e513a458/keps/sig-instrumentation/20181106-kubernetes-metrics-overhaul.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83838",
-    "pr_number": 83838,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83839": {
-    "commit": "1ae9713f7f3c8c1bb595a9c04a2f8518c3e5902f",
-    "text": "Deprecated metric `kubeproxy_sync_proxy_rules_latency_microseconds` has been turned off.",
-    "markdown": "Deprecated metric `kubeproxy_sync_proxy_rules_latency_microseconds` has been turned off. ([#83839](https://github.com/kubernetes/kubernetes/pull/83839), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Network",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/da4b7050ccae7a947e4d60f94ab28513e513a458/keps/sig-instrumentation/20181106-kubernetes-metrics-overhaul.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83839",
-    "pr_number": 83839,
-    "kinds": ["feature"],
-    "sigs": ["network"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83902": {
-    "commit": "78abdf53750d9c76a9d6afaf928fe1387cf06a98",
-    "text": "clean duplicate GetPodServiceMemberships function",
-    "markdown": "clean duplicate GetPodServiceMemberships function ([#83902](https://github.com/kubernetes/kubernetes/pull/83902), [@gongguan](https://github.com/gongguan))",
-    "author": "gongguan",
-    "author_url": "https://github.com/gongguan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83902",
-    "pr_number": 83902,
-    "kinds": ["cleanup"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83907": {
-    "commit": "83c6bf361d6a060181e3e5c0d1e7cf1e3dc3da7b",
-    "text": "Bumps metrics-server version to v0.3.6 with following bugfix:\n\n-  Don't break metric storage when duplicate pod metrics encountered causing hpa to fail",
-    "markdown": "Bumps metrics-server version to v0.3.6 with following bugfix:\n  \n  -  Don't break metric storage when duplicate pod metrics encountered causing hpa to fail ([#83907](https://github.com/kubernetes/kubernetes/pull/83907), [@olagacek](https://github.com/olagacek))",
-    "author": "olagacek",
-    "author_url": "https://github.com/olagacek",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83907",
-    "pr_number": 83907,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "83910": {
-    "commit": "da552df5a06cc476c31a0a8772c77fe375a4bf57",
-    "text": "Allow dynamically set glog logging level of kube-scheduler",
-    "markdown": "Allow dynamically set glog logging level of kube-scheduler ([#83910](https://github.com/kubernetes/kubernetes/pull/83910), [@mrkm4ntr](https://github.com/mrkm4ntr))\n\n  Courtesy of SIG Scheduling",
-    "author": "mrkm4ntr",
-    "author_url": "https://github.com/mrkm4ntr",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83910",
-    "pr_number": 83910,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83911": {
-    "commit": "634bddddf271868ed8c0a9fbd853d62e0c0446bb",
-    "text": "Fixed an issue with informers missing an `Added` event if a recently deleted object was immediately recreated at the same time the informer dropped a watch and relisted.",
-    "markdown": "Fixed an issue with informers missing an `Added` event if a recently deleted object was immediately recreated at the same time the informer dropped a watch and relisted. ([#83911](https://github.com/kubernetes/kubernetes/pull/83911), [@matte21](https://github.com/matte21))",
-    "author": "matte21",
-    "author_url": "https://github.com/matte21",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83911",
-    "pr_number": 83911,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "83932": {
-    "commit": "b4d3dc371889452f9be22eaa4617e4da45bf2931",
-    "text": "kubeadm: enhance certs check-expiration to show the expiration info of related CAs",
-    "markdown": "kubeadm: enhance certs check-expiration to show the expiration info of related CAs ([#83932](https://github.com/kubernetes/kubernetes/pull/83932), [@SataQiu](https://github.com/SataQiu))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83932",
-    "pr_number": 83932,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83944": {
-    "commit": "57353d524253fed5ef267101d47c230cc05a4894",
-    "text": "Added the `crictl` Windows binaries as well as the Linux 32bit binary to the release archives",
-    "markdown": "Added the `crictl` Windows binaries as well as the Linux 32bit binary to the release archives ([#83944](https://github.com/kubernetes/kubernetes/pull/83944), [@saschagrunert](https://github.com/saschagrunert))",
-    "documentation": [
-      {
-        "description": "crictl release archives",
-        "url": "https://github.com/kubernetes-sigs/cri-tools/releases/tag/v1.16.1",
-        "type": "external"
-      }
-    ],
-    "author": "saschagrunert",
-    "author_url": "https://github.com/saschagrunert",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83944",
-    "pr_number": 83944,
-    "areas": ["release-eng"],
-    "kinds": ["cleanup"],
-    "sigs": ["release"],
-    "release_version": "1.17.0"
-  },
-  "83945": {
-    "commit": "cd7a0ee8eba749551366c082c15e54aed407d101",
-    "text": "client-go: improved allocation behavior of the delaying workqueue when handling objects with far-future ready times.",
-    "markdown": "client-go: improved allocation behavior of the delaying workqueue when handling objects with far-future ready times. ([#83945](https://github.com/kubernetes/kubernetes/pull/83945), [@barkbay](https://github.com/barkbay))",
-    "author": "barkbay",
-    "author_url": "https://github.com/barkbay",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83945",
-    "pr_number": 83945,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "83950": {
-    "commit": "4b58ef0e14cc3eead5954b9d08313e164efba2a7",
-    "text": "kubeadm no longer removes /etc/cni/net.d as it does not install it. Users should remove files from it manually or rely on the component that created them",
-    "markdown": "kubeadm no longer removes /etc/cni/net.d as it does not install it. Users should remove files from it manually or rely on the component that created them ([#83950](https://github.com/kubernetes/kubernetes/pull/83950), [@yastij](https://github.com/yastij))",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83950",
-    "pr_number": 83950,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "83951": {
-    "commit": "064458de46189749da9102015312b857c47c963d",
-    "text": "Fix handling tombstones in pod-disruption-budged controller.",
-    "markdown": "Fix handling tombstones in pod-disruption-budged controller. ([#83951](https://github.com/kubernetes/kubernetes/pull/83951), [@zouyee](https://github.com/zouyee))",
-    "author": "zouyee",
-    "author_url": "https://github.com/zouyee",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83951",
-    "pr_number": 83951,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.17.0"
-  },
-  "83956": {
-    "commit": "554412c62cf84a8c096399d61bb3837d00b82966",
-    "text": "Switched intstr.Type to sized integer to follow API guidelines and improve compatibility with proto libraries",
-    "markdown": "Switched intstr.Type to sized integer to follow API guidelines and improve compatibility with proto libraries ([#83956](https://github.com/kubernetes/kubernetes/pull/83956), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83956",
-    "pr_number": 83956,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "83963": {
-    "commit": "8d8a0684866d75aa195de20871a6164794b11ebc",
-    "text": "Scheduler policy configs can no longer be declared multiple times",
-    "markdown": "Scheduler policy configs can no longer be declared multiple times ([#83963](https://github.com/kubernetes/kubernetes/pull/83963), [@damemi](https://github.com/damemi))",
-    "author": "damemi",
-    "author_url": "https://github.com/damemi",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83963",
-    "pr_number": 83963,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.17.0"
-  },
-  "83965": {
-    "commit": "5dc87d2919818bbfd5c68eb1aed766b849a9062c",
-    "text": "Adds a new label to indicate what is managing an EndpointSlice.",
-    "markdown": "Adds a new label to indicate what is managing an EndpointSlice. ([#83965](https://github.com/kubernetes/kubernetes/pull/83965), [@robscott](https://github.com/robscott))\n\n  Courtesy of SIG Apps, and SIG Network",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83965",
-    "pr_number": 83965,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["apps", "network"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "83987": {
-    "commit": "09f453ff8322979ed5a7611bc2e5528506c1fc7f",
-    "text": "Update etcd client side to v3.4.3\nDeprecated prometheus request meta-metrics have been removed (http_request_duration_microseconds, http_request_duration_microseconds_sum, http_request_duration_microseconds_count, http_request_size_bytes, http_request_size_bytes_sum, http_request_size_bytes_count, http_requests_total, http_response_size_bytes, http_response_size_bytes_sum, http_response_size_bytes_count) due to removal from the prometheus client library. Prometheus http request meta-metrics are now generated from [promhttp.InstrumentMetricHandler](https://godoc.org/github.com/prometheus/client_golang/prometheus/promhttp\u0026#35;InstrumentMetricHandler) instead.",
-    "markdown": "Update etcd client side to v3.4.3\n  Deprecated prometheus request meta-metrics have been removed (http_request_duration_microseconds, http_request_duration_microseconds_sum, http_request_duration_microseconds_count, http_request_size_bytes, http_request_size_bytes_sum, http_request_size_bytes_count, http_requests_total, http_response_size_bytes, http_response_size_bytes_sum, http_response_size_bytes_count) due to removal from the prometheus client library. Prometheus http request meta-metrics are now generated from [promhttp.InstrumentMetricHandler](https://godoc.org/github.com/prometheus/client_golang/prometheus/promhttp\u0026#35;InstrumentMetricHandler) instead. ([#83987](https://github.com/kubernetes/kubernetes/pull/83987), [@wenjiaswe](https://github.com/wenjiaswe))",
-    "author": "wenjiaswe",
-    "author_url": "https://github.com/wenjiaswe",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83987",
-    "pr_number": 83987,
-    "areas": [
-      "apiserver",
-      "cloudprovider",
-      "code-generation",
-      "dependency",
-      "e2e-test-framework",
-      "kubeadm",
-      "kubectl",
-      "kubelet",
-      "test"
-    ],
-    "kinds": ["api-change"],
-    "sigs": [
-      "api-machinery",
-      "apps",
-      "auth",
-      "autoscaling",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "network",
-      "node",
-      "scheduling",
-      "storage",
-      "testing"
-    ],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84007": {
-    "commit": "8ff16f35f8fd2cc504fba25be9d45f0e7dce5da3",
-    "text": "Reduce default NodeStatusReportFrequency to 5 minutes. With this change, periodic node status updates will be send every 5m if node status doesn't change (otherwise they are still send with 10s).\n\nBump NodeProblemDetector version to v0.8.0 to reduce forced NodeStatus updates frequency to 5 minutes.",
-    "markdown": "Reduce default NodeStatusReportFrequency to 5 minutes. With this change, periodic node status updates will be send every 5m if node status doesn't change (otherwise they are still send with 10s).\n  \n  Bump NodeProblemDetector version to v0.8.0 to reduce forced NodeStatus updates frequency to 5 minutes. ([#84007](https://github.com/kubernetes/kubernetes/pull/84007), [@wojtek-t](https://github.com/wojtek-t))\n\n  Courtesy of SIG Cluster Lifecycle, SIG Node, SIG Scalability, and SIG Testing",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84007",
-    "pr_number": 84007,
-    "areas": ["kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle", "node", "scalability", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84011": {
-    "commit": "4fdf5826877c655041b01764c3447b154bff7c33",
-    "text": "Add permit_wait_duration_seconds metric for scheduler.",
-    "markdown": "Add permit_wait_duration_seconds metric for scheduler. ([#84011](https://github.com/kubernetes/kubernetes/pull/84011), [@liu-cong](https://github.com/liu-cong))\n\n  Courtesy of SIG Scheduling",
-    "author": "liu-cong",
-    "author_url": "https://github.com/liu-cong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84011",
-    "pr_number": 84011,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84018": {
-    "commit": "ec63e099ba5b08560b20f922bd610a15bcfff2cc",
-    "text": "Update to Ingress-GCE v1.6.1",
-    "markdown": "Update to Ingress-GCE v1.6.1 ([#84018](https://github.com/kubernetes/kubernetes/pull/84018), [@rramkumar1](https://github.com/rramkumar1))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "rramkumar1",
-    "author_url": "https://github.com/rramkumar1",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84018",
-    "pr_number": 84018,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84038": {
-    "commit": "54a30700a38452a5113adcfba0f98adcc5e05f2d",
-    "text": "Update Cluster Autoscaler version to 1.16.2 (CA release docs: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.16.2)",
-    "markdown": "Update Cluster Autoscaler version to 1.16.2 (CA release docs: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.16.2) ([#84038](https://github.com/kubernetes/kubernetes/pull/84038), [@losipiuk](https://github.com/losipiuk))",
-    "author": "losipiuk",
-    "author_url": "https://github.com/losipiuk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84038",
-    "pr_number": 84038,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "84042": {
-    "commit": "6ff3b68d72089ce4fd0360fca7c1a4f05df834dc",
-    "text": "Removed dependency on kubectl from several storage E2E tests",
-    "markdown": "Removed dependency on kubectl from several storage E2E tests ([#84042](https://github.com/kubernetes/kubernetes/pull/84042), [@okartau](https://github.com/okartau))\n\n  Courtesy of SIG Storage, and SIG Testing",
-    "author": "okartau",
-    "author_url": "https://github.com/okartau",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84042",
-    "pr_number": 84042,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["feature"],
-    "sigs": ["storage", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84049": {
-    "commit": "4984c6f00033c5649710d549d5917623d2cf3cc1",
-    "text": "Fixed binding of block PersistentVolumes / PersistentVolumeClaims when BlockVolume feature is off.",
-    "markdown": "Fixed binding of block PersistentVolumes / PersistentVolumeClaims when BlockVolume feature is off. ([#84049](https://github.com/kubernetes/kubernetes/pull/84049), [@jsafrane](https://github.com/jsafrane))\n\n  Courtesy of SIG Apps, and SIG Storage",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/b1b2751/keps/sig-storage/20191008-raw-block-support.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84049",
-    "pr_number": 84049,
-    "kinds": ["feature"],
-    "sigs": ["apps", "storage"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84064": {
-    "commit": "578d0616669b53be13df0bc8805daf64ae9d01e3",
-    "text": "Update to use go1.12.12",
-    "markdown": "Update to use go1.12.12 ([#84064](https://github.com/kubernetes/kubernetes/pull/84064), [@cblecker](https://github.com/cblecker))",
-    "author": "cblecker",
-    "author_url": "https://github.com/cblecker",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84064",
-    "pr_number": 84064,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["release", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84084": {
-    "commit": "badcd4af3f592376ce891b7c1b7a43ed6a18a348",
-    "text": "Migrate controller-manager and scheduler to EndpointsLeases leader election.",
-    "markdown": "Migrate controller-manager and scheduler to EndpointsLeases leader election. ([#84084](https://github.com/kubernetes/kubernetes/pull/84084), [@wojtek-t](https://github.com/wojtek-t))",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84084",
-    "pr_number": 84084,
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "apps", "auth", "cloud-provider", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84091": {
-    "commit": "a7cc252b12c59d73a5e89dd03dba7c878a4a1170",
-    "text": "Adds FQDN addressType support for EndpointSlice.",
-    "markdown": "Adds FQDN addressType support for EndpointSlice. ([#84091](https://github.com/kubernetes/kubernetes/pull/84091), [@robscott](https://github.com/robscott))\n\n  Courtesy of SIG API Machinery, and SIG Network",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84091",
-    "pr_number": 84091,
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "network"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84113": {
-    "commit": "e1dc959881508f64c5f8a2e6018ed49fa7c619b4",
-    "text": "Introduce x-kubernetes-map-type annotation as a CRD API extension. Enables this particular validation for server-side apply.",
-    "markdown": "Introduce x-kubernetes-map-type annotation as a CRD API extension. Enables this particular validation for server-side apply. ([#84113](https://github.com/kubernetes/kubernetes/pull/84113), [@enxebre](https://github.com/enxebre))",
-    "author": "enxebre",
-    "author_url": "https://github.com/enxebre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84113",
-    "pr_number": 84113,
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "84118": {
-    "commit": "9648d56765bdd77aa504910cdc1795189f11cd51",
-    "text": "kubeadm: add a new \"kubelet-finalize\" phase as part of the \"init\" workflow and an experimental sub-phase to enable automatic kubelet client certificate rotation on primary control-plane nodes.\n\nPrior to 1.17 and for existing nodes created by \"kubeadm init\" where kubelet client certificate rotation is desired, you must modify \"/etc/kubernetes/kubelet.conf\" to point to the PEM symlink for rotation:\n\"client-certificate: /var/lib/kubelet/pki/kubelet-client-current.pem\" and \"client-key: /var/lib/kubelet/pki/kubelet-client-current.pem\", replacing the embedded client certificate and key.",
-    "markdown": "kubeadm: add a new \"kubelet-finalize\" phase as part of the \"init\" workflow and an experimental sub-phase to enable automatic kubelet client certificate rotation on primary control-plane nodes.\n  \n  Prior to 1.17 and for existing nodes created by \"kubeadm init\" where kubelet client certificate rotation is desired, you must modify \"/etc/kubernetes/kubelet.conf\" to point to the PEM symlink for rotation:\n  \"client-certificate: /var/lib/kubelet/pki/kubelet-client-current.pem\" and \"client-key: /var/lib/kubelet/pki/kubelet-client-current.pem\", replacing the embedded client certificate and key. ([#84118](https://github.com/kubernetes/kubernetes/pull/84118), [@neolit123](https://github.com/neolit123))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84118",
-    "pr_number": 84118,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "action_required": true,
-    "release_version": "1.17.0"
-  },
-  "84120": {
-    "commit": "1d489b77e541aecd55f641ebac888c3a3b940fd8",
-    "text": "A new `kubelet_preemptions` metric is reported from Kubelets to track the number of preemptions occuring over time, and which resource is triggering those preemptions.",
-    "markdown": "A new `kubelet_preemptions` metric is reported from Kubelets to track the number of preemptions occuring over time, and which resource is triggering those preemptions. ([#84120](https://github.com/kubernetes/kubernetes/pull/84120), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84120",
-    "pr_number": 84120,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["instrumentation", "node", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84129": {
-    "commit": "3e5f6bdaa48723a91a208650e8d2d601fc1a55a9",
-    "text": "kube-scheduler: emits a warning when a malformed component config file is used with v1alpha1.",
-    "markdown": "kube-scheduler: emits a warning when a malformed component config file is used with v1alpha1. ([#84129](https://github.com/kubernetes/kubernetes/pull/84129), [@obitech](https://github.com/obitech))",
-    "author": "obitech",
-    "author_url": "https://github.com/obitech",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84129",
-    "pr_number": 84129,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84138": {
-    "commit": "9790262f128aa9d27733dac46619af1d18459439",
-    "text": "None.",
-    "markdown": "None. ([#84138](https://github.com/kubernetes/kubernetes/pull/84138), [@nilo19](https://github.com/nilo19))",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84138",
-    "pr_number": 84138,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.17.0"
-  },
-  "84143": {
-    "commit": "7fd399e4be691c32fc413c380d67656123754780",
-    "text": "kube-proxy: emits a warning when a malformed component config file is used with v1alpha1.",
-    "markdown": "kube-proxy: emits a warning when a malformed component config file is used with v1alpha1. ([#84143](https://github.com/kubernetes/kubernetes/pull/84143), [@phenixblue](https://github.com/phenixblue))",
-    "author": "phenixblue",
-    "author_url": "https://github.com/phenixblue",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84143",
-    "pr_number": 84143,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.17.0"
-  },
-  "84148": {
-    "commit": "ffffd6b6f55d15c6868bc2080419740dc7a0d93d",
-    "text": "filter plugin for cloud provider storage predicate",
-    "markdown": "filter plugin for cloud provider storage predicate ([#84148](https://github.com/kubernetes/kubernetes/pull/84148), [@gongguan](https://github.com/gongguan))\n\n  Courtesy of SIG Scheduling, and SIG Testing",
-    "author": "gongguan",
-    "author_url": "https://github.com/gongguan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84148",
-    "pr_number": 84148,
-    "areas": ["test"],
-    "kinds": ["feature"],
-    "sigs": ["scheduling", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84152": {
-    "commit": "97c7ccc54729e7ab31775902697c21663f3882b0",
-    "text": "TaintNodesByCondition was graduated to GA, CheckNodeMemoryPressure, CheckNodePIDPressure, CheckNodeDiskPressure, CheckNodeCondition were accidentally removed since 1.12, the replacement is to use CheckNodeUnschedulablePred",
-    "markdown": "TaintNodesByCondition was graduated to GA, CheckNodeMemoryPressure, CheckNodePIDPressure, CheckNodeDiskPressure, CheckNodeCondition were accidentally removed since 1.12, the replacement is to use CheckNodeUnschedulablePred ([#84152](https://github.com/kubernetes/kubernetes/pull/84152), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84152",
-    "pr_number": 84152,
-    "kinds": ["cleanup"],
-    "sigs": ["scheduling"],
-    "release_version": "1.17.0"
-  },
-  "84156": {
-    "commit": "6d314f860ebd0ea5b7c6462aa65347741f2747e8",
-    "text": "Fix kubelet metrics gathering on non-English Windows hosts",
-    "markdown": "Fix kubelet metrics gathering on non-English Windows hosts ([#84156](https://github.com/kubernetes/kubernetes/pull/84156), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84156",
-    "pr_number": 84156,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "windows"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84173": {
-    "commit": "24ae4d6718f679e6331097811ae77fcbf275e124",
-    "text": "local: support local volume block mode reconstruction",
-    "markdown": "local: support local volume block mode reconstruction ([#84173](https://github.com/kubernetes/kubernetes/pull/84173), [@cofyc](https://github.com/cofyc))",
-    "author": "cofyc",
-    "author_url": "https://github.com/cofyc",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84173",
-    "pr_number": 84173,
-    "areas": ["kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["node", "storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84181": {
-    "commit": "5f03d33fc938e792b66ce1a1062b69a075194d4d",
-    "text": "Reduced frequency of DescribeVolumes calls of AWS API when attaching/detaching a volume.",
-    "markdown": "Reduced frequency of DescribeVolumes calls of AWS API when attaching/detaching a volume. ([#84181](https://github.com/kubernetes/kubernetes/pull/84181), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84181",
-    "pr_number": 84181,
-    "areas": ["cloudprovider"],
-    "kinds": ["flake"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84194": {
-    "commit": "b9030ff666772b76c6bb96c147a52341a9be94b9",
-    "text": "Fix incorrect network policy description suggesting that pods are isolated when a network policy has no rules of a given type",
-    "markdown": "Fix incorrect network policy description suggesting that pods are isolated when a network policy has no rules of a given type ([#84194](https://github.com/kubernetes/kubernetes/pull/84194), [@jackkleeman](https://github.com/jackkleeman))",
-    "author": "jackkleeman",
-    "author_url": "https://github.com/jackkleeman",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84194",
-    "pr_number": 84194,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli", "network"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84200": {
-    "commit": "04632e84e1348eb298027bae917cfede920c10a2",
-    "text": "Reload apiserver serving certificate from disk every minute",
-    "markdown": "Reload apiserver serving certificate from disk every minute ([#84200](https://github.com/kubernetes/kubernetes/pull/84200), [@jackkleeman](https://github.com/jackkleeman))\n\n  Courtesy of SIG API Machinery, SIG Auth, SIG Node, and SIG Testing",
-    "author": "jackkleeman",
-    "author_url": "https://github.com/jackkleeman",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84200",
-    "pr_number": 84200,
-    "areas": ["apiserver", "kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "auth", "node", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84207": {
-    "commit": "ed19c74f39550ddb56d5aad0702f8b5452b40efa",
-    "text": "EndpointSlice hostname is now set in the same conditions Endpoints hostname is.",
-    "markdown": "EndpointSlice hostname is now set in the same conditions Endpoints hostname is. ([#84207](https://github.com/kubernetes/kubernetes/pull/84207), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84207",
-    "pr_number": 84207,
-    "kinds": ["bug"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84218": {
-    "commit": "ed5b03831385b0071b127ff6c329ae344b9780f6",
-    "text": "local: support local filesystem volume with block resource reconstruction",
-    "markdown": "local: support local filesystem volume with block resource reconstruction ([#84218](https://github.com/kubernetes/kubernetes/pull/84218), [@cofyc](https://github.com/cofyc))\n\n  Courtesy of SIG Node, SIG Storage, and SIG Testing",
-    "author": "cofyc",
-    "author_url": "https://github.com/cofyc",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84218",
-    "pr_number": 84218,
-    "areas": ["kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["node", "storage", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84260": {
-    "commit": "06252a4630f5c52d8abfc3b3e36c55964d106650",
-    "text": "Pod labels can no longer be updated through the pod/status updates by nodes.",
-    "markdown": "Pod labels can no longer be updated through the pod/status updates by nodes. ([#84260](https://github.com/kubernetes/kubernetes/pull/84260), [@tallclair](https://github.com/tallclair))\n\n  Courtesy of SIG Auth, and SIG Node",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-auth/20190916-noderestriction-pods.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84260",
-    "pr_number": 84260,
-    "kinds": ["feature"],
-    "sigs": ["auth", "node"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84264": {
-    "commit": "2905275a08f8e3999e7e6bc8b02f407fd6a0ff36",
-    "text": "Optimize inter-pod affinity preferredDuringSchedulingIgnoredDuringExecution type, up to 4x in some cases.",
-    "markdown": "Optimize inter-pod affinity preferredDuringSchedulingIgnoredDuringExecution type, up to 4x in some cases. ([#84264](https://github.com/kubernetes/kubernetes/pull/84264), [@ahg-g](https://github.com/ahg-g))\n\n  Courtesy of SIG Scheduling",
-    "author": "ahg-g",
-    "author_url": "https://github.com/ahg-g",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84264",
-    "pr_number": 84264,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84280": {
-    "commit": "24fb2c1afd51069a526e4c36ea5d3af993fd6b26",
-    "text": "Improving the performance of Endpoint and EndpointSlice controllers by caching Service Selectors",
-    "markdown": "Improving the performance of Endpoint and EndpointSlice controllers by caching Service Selectors ([#84280](https://github.com/kubernetes/kubernetes/pull/84280), [@gongguan](https://github.com/gongguan))",
-    "author": "gongguan",
-    "author_url": "https://github.com/gongguan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84280",
-    "pr_number": 84280,
-    "kinds": ["bug"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84282": {
-    "commit": "a3560d3ad9a7e2deb7d8b7e9e54081e7cbbac0d1",
-    "text": "The built-in system:csi-external-provisioner and system:csi-external-attacher cluster roles are removed as of 1.17 release",
-    "markdown": "The built-in system:csi-external-provisioner and system:csi-external-attacher cluster roles are removed as of 1.17 release ([#84282](https://github.com/kubernetes/kubernetes/pull/84282), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84282",
-    "pr_number": 84282,
-    "kinds": ["cleanup"],
-    "sigs": ["auth", "storage"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84292": {
-    "commit": "21df24cc067f4fd6192cb82992fef0f925b48d19",
-    "text": "New flag `--show-hidden-metrics-for-version` in kube-apiserver can be used to show all hidden metrics that deprecated in the previous minor release.",
-    "markdown": "New flag `--show-hidden-metrics-for-version` in kube-apiserver can be used to show all hidden metrics that deprecated in the previous minor release. ([#84292](https://github.com/kubernetes/kubernetes/pull/84292), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG API Machinery, SIG Cluster Lifecycle, and SIG Instrumentation",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84292",
-    "pr_number": 84292,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "instrumentation"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84297": {
-    "commit": "41730db3f0637a9175a41a435b2a6e7051f73f2b",
-    "text": "User can now use component config to configure NodeLabel plugin for the scheduler framework.",
-    "markdown": "User can now use component config to configure NodeLabel plugin for the scheduler framework. ([#84297](https://github.com/kubernetes/kubernetes/pull/84297), [@liu-cong](https://github.com/liu-cong))\n\n  Courtesy of SIG Scheduling",
-    "author": "liu-cong",
-    "author_url": "https://github.com/liu-cong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84297",
-    "pr_number": 84297,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84303": {
-    "commit": "c7bb076ade3e91cc13a0e855e155484fb4040191",
-    "text": "Reload apiserver SNI certificates from disk every minute",
-    "markdown": "Reload apiserver SNI certificates from disk every minute ([#84303](https://github.com/kubernetes/kubernetes/pull/84303), [@jackkleeman](https://github.com/jackkleeman))\n\n  Courtesy of SIG API Machinery, and SIG Testing",
-    "author": "jackkleeman",
-    "author_url": "https://github.com/jackkleeman",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84303",
-    "pr_number": 84303,
-    "areas": ["apiserver", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84304": {
-    "commit": "f1e912c38abcecfb64e36eb161128b15e69a135b",
-    "text": "`--runtime-config` now supports an `api/beta=false` value which disables all built-in REST API versions matching `v[0-9]+beta[0-9]+`.\n`--feature-gates` now supports an `AllBeta=false` value which disables all beta feature gates.",
-    "markdown": "`--runtime-config` now supports an `api/beta=false` value which disables all built-in REST API versions matching `v[0-9]+beta[0-9]+`.\n  `--feature-gates` now supports an `AllBeta=false` value which disables all beta feature gates. ([#84304](https://github.com/kubernetes/kubernetes/pull/84304), [@liggitt](https://github.com/liggitt))\n\n  Courtesy of SIG API Machinery, and SIG Cluster Lifecycle",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/pull/1332",
-        "type": "KEP"
-      }
-    ],
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84304",
-    "pr_number": 84304,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84321": {
-    "commit": "cb0f80efe7982741e19dad9f1a9fa91797f57ed3",
-    "text": "CSI detach timeout increased from 10 seconds to 2 minutes",
-    "markdown": "CSI detach timeout increased from 10 seconds to 2 minutes ([#84321](https://github.com/kubernetes/kubernetes/pull/84321), [@cduchesne](https://github.com/cduchesne))",
-    "author": "cduchesne",
-    "author_url": "https://github.com/cduchesne",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84321",
-    "pr_number": 84321,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.17.0"
-  },
-  "84329": {
-    "commit": "e1d3cc74e554f37765e81fdf265cd1318dc451ae",
-    "text": "Update default etcd server version to 3.4.3",
-    "markdown": "Update default etcd server version to 3.4.3 ([#84329](https://github.com/kubernetes/kubernetes/pull/84329), [@jingyih](https://github.com/jingyih))",
-    "author": "jingyih",
-    "author_url": "https://github.com/jingyih",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84329",
-    "pr_number": 84329,
-    "areas": ["e2e-test-framework", "etcd", "kubeadm", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84351": {
-    "commit": "ae1536835545c8d9fdb896df10cc3439c3242445",
-    "text": "Promote NodeLease feature to GA.\nThe feature make Lease object changes an additional healthiness signal from Node. Together with that, we reduce frequency of NodeStatus updates to 5m by default in case of no changes to status itself",
-    "markdown": "Promote NodeLease feature to GA.\n  The feature make Lease object changes an additional healthiness signal from Node. Together with that, we reduce frequency of NodeStatus updates to 5m by default in case of no changes to status itself ([#84351](https://github.com/kubernetes/kubernetes/pull/84351), [@wojtek-t](https://github.com/wojtek-t))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG Node, and SIG Testing",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84351",
-    "pr_number": 84351,
-    "areas": ["kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "apps", "auth", "node", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84356": {
-    "commit": "c10d29ba8890d37ddf316a4840860d039a5ab2bc",
-    "text": "Pod process namespace sharing is now Generally Available. The `PodShareProcessNamespace` feature gate is now deprecated and will be removed in Kubernetes 1.19.",
-    "markdown": "Pod process namespace sharing is now Generally Available. The `PodShareProcessNamespace` feature gate is now deprecated and will be removed in Kubernetes 1.19. ([#84356](https://github.com/kubernetes/kubernetes/pull/84356), [@verb](https://github.com/verb))\n\n  Courtesy of SIG Apps, SIG Node, and SIG Testing",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://git.k8s.io/enhancements/keps/sig-node/20190920-pod-pid-namespace.md",
-        "type": "external"
-      },
-      {
-        "description": "[Usage]",
-        "url": "https://kubernetes.io/docs/tasks/configure-pod-container/share-process-namespace",
-        "type": "official"
-      }
-    ],
-    "author": "verb",
-    "author_url": "https://github.com/verb",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84356",
-    "pr_number": 84356,
-    "areas": ["kubelet", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["apps", "node", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84365": {
-    "commit": "08410cbf06f1c8649157407c660b1c5cb528947b",
-    "text": "Removed Alpha feature `MountContainers`",
-    "markdown": "Removed Alpha feature `MountContainers` ([#84365](https://github.com/kubernetes/kubernetes/pull/84365), [@codenrhoden](https://github.com/codenrhoden))",
-    "author": "codenrhoden",
-    "author_url": "https://github.com/codenrhoden",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84365",
-    "pr_number": 84365,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node", "storage"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84383": {
-    "commit": "6170296b66b3660f2f2f62644c3cd0d8c215f3c1",
-    "text": "This PR sets the --cluster-dns flag value to kube-dns service IP whether or not NodeLocal DNSCache is enabled. NodeLocal DNSCache will listen on both the link-local as well as the service IP.",
-    "markdown": "This PR sets the --cluster-dns flag value to kube-dns service IP whether or not NodeLocal DNSCache is enabled. NodeLocal DNSCache will listen on both the link-local as well as the service IP. ([#84383](https://github.com/kubernetes/kubernetes/pull/84383), [@prameshj](https://github.com/prameshj))",
-    "author": "prameshj",
-    "author_url": "https://github.com/prameshj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84383",
-    "pr_number": 84383,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84390": {
-    "commit": "64f4be5b328a4df8a709b95604743086013a70e4",
-    "text": "EndpointSlices are now beta and enabled by default for better Network Endpoint performance at scale.",
-    "markdown": "EndpointSlices are now beta and enabled by default for better Network Endpoint performance at scale. ([#84390](https://github.com/kubernetes/kubernetes/pull/84390), [@robscott](https://github.com/robscott))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Auth, SIG CLI, SIG Network, and SIG Testing",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84390",
-    "pr_number": 84390,
-    "areas": ["apiserver", "ipvs", "kubectl", "test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "auth", "cli", "network", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84421": {
-    "commit": "4002e4c7e7b12b052310bce808117d2f2472ad4b",
-    "text": "Fix the bug that EndpointSlice for masters wasn't created after enabling EndpointSlice feature on a pre-existing cluster.",
-    "markdown": "Fix the bug that EndpointSlice for masters wasn't created after enabling EndpointSlice feature on a pre-existing cluster. ([#84421](https://github.com/kubernetes/kubernetes/pull/84421), [@tnqn](https://github.com/tnqn))",
-    "author": "tnqn",
-    "author_url": "https://github.com/tnqn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84421",
-    "pr_number": 84421,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "84468": {
-    "commit": "b6c8f4916dc3a625e9decc2fd273e2a522e8b474",
-    "text": "kubeadm: always mount the kube-controller-manager hostPath volume that is given by the --flex-volume-plugin-dir flag.",
-    "markdown": "kubeadm: always mount the kube-controller-manager hostPath volume that is given by the --flex-volume-plugin-dir flag. ([#84468](https://github.com/kubernetes/kubernetes/pull/84468), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84468",
-    "pr_number": 84468,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "84472": {
-    "commit": "42273a4dbf27cd2ed362c1b2e418d2e15fe2c53c",
-    "text": "Adds Windows Server build information as a label on the node.",
-    "markdown": "Adds Windows Server build information as a label on the node. ([#84472](https://github.com/kubernetes/kubernetes/pull/84472), [@gab-satchi](https://github.com/gab-satchi))\n\n  Courtesy of SIG Node, and SIG Windows",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/pull/1302",
-        "type": "KEP"
-      }
-    ],
-    "author": "gab-satchi",
-    "author_url": "https://github.com/gab-satchi",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84472",
-    "pr_number": 84472,
-    "areas": ["kubelet"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["node", "windows"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84476": {
-    "commit": "28887de8cd1b51ba2416a2e60b0b023728c5a54e",
-    "text": "update the latest validated version of Docker to 19.03",
-    "markdown": "update the latest validated version of Docker to 19.03 ([#84476](https://github.com/kubernetes/kubernetes/pull/84476), [@neolit123](https://github.com/neolit123))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84476",
-    "pr_number": 84476,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84481": {
-    "commit": "d0f2c7c5902dbeb934e52fc925b3cc0488440ee2",
-    "text": "Fixed EndpointSlice port name validation to match Endpoint port name validation (allowing port names longer than 15 characters)",
-    "markdown": "Fixed EndpointSlice port name validation to match Endpoint port name validation (allowing port names longer than 15 characters) ([#84481](https://github.com/kubernetes/kubernetes/pull/84481), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84481",
-    "pr_number": 84481,
-    "kinds": ["api-change", "bug"],
-    "sigs": ["network"],
-    "release_version": "1.17.0"
-  },
-  "84485": {
-    "commit": "0155d18fbc798031fe904bc99abe2b935058beb8",
-    "text": "Mirror pods now include an ownerReference for the node that created them.",
-    "markdown": "Mirror pods now include an ownerReference for the node that created them. ([#84485](https://github.com/kubernetes/kubernetes/pull/84485), [@tallclair](https://github.com/tallclair))\n\n  Courtesy of SIG Node, and SIG Testing",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-auth/20190916-noderestriction-pods.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "tallclair",
-    "author_url": "https://github.com/tallclair",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84485",
-    "pr_number": 84485,
-    "areas": ["kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["node", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84522": {
-    "commit": "ce11622bb682027ddd0f195d539d6de7bea2fd01",
-    "text": "Add plugin_execution_duration_seconds metric for scheduler framework plugins.",
-    "markdown": "Add plugin_execution_duration_seconds metric for scheduler framework plugins. ([#84522](https://github.com/kubernetes/kubernetes/pull/84522), [@liu-cong](https://github.com/liu-cong))\n\n  Courtesy of SIG Scheduling",
-    "author": "liu-cong",
-    "author_url": "https://github.com/liu-cong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84522",
-    "pr_number": 84522,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84524": {
-    "commit": "1ab6360997dda691d6671fc547abbe3776ae8590",
-    "text": "A new flag \"progress-report-url\" has been added to the test context which allows progress information about the test run to be sent to a webhook. In addition, this information is printed to stdout to aid in users watching the logs.",
-    "markdown": "A new flag \"progress-report-url\" has been added to the test context which allows progress information about the test run to be sent to a webhook. In addition, this information is printed to stdout to aid in users watching the logs. ([#84524](https://github.com/kubernetes/kubernetes/pull/84524), [@johnSchnake](https://github.com/johnSchnake))\n\n  Courtesy of SIG Testing",
-    "author": "johnSchnake",
-    "author_url": "https://github.com/johnSchnake",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84524",
-    "pr_number": 84524,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["feature"],
-    "sigs": ["testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84533": {
-    "commit": "897ce3073c056ef2b94708cdd1b56522b6598b54",
-    "text": "BREAKING CHANGE: Remove plugin watching of deprecated directory {kubelet_root_dir}/plugins and CSI V0 support in accordance with deprecation announcement in https://v1-13.docs.kubernetes.io/docs/setup/release/notes/",
-    "markdown": "BREAKING CHANGE: Remove plugin watching of deprecated directory {kubelet_root_dir}/plugins and CSI V0 support in accordance with deprecation announcement in https://v1-13.docs.kubernetes.io/docs/setup/release/notes/ ([#84533](https://github.com/kubernetes/kubernetes/pull/84533), [@davidz627](https://github.com/davidz627))",
-    "author": "davidz627",
-    "author_url": "https://github.com/davidz627",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84533",
-    "pr_number": 84533,
-    "areas": ["dependency", "kubelet", "test"],
-    "kinds": ["cleanup", "documentation"],
-    "sigs": ["node", "storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84534": {
-    "commit": "0ed66351739608660fef27de55d8e0dff6570f28",
-    "text": "Added kubelet serving certificate metric `server_rotation_seconds` which is a histogram reporting the age of a just rotated serving certificate in seconds.",
-    "markdown": "Added kubelet serving certificate metric `server_rotation_seconds` which is a histogram reporting the age of a just rotated serving certificate in seconds. ([#84534](https://github.com/kubernetes/kubernetes/pull/84534), [@sambdavidson](https://github.com/sambdavidson))\n\n  Courtesy of SIG API Machinery, SIG Auth, SIG Instrumentation, and SIG Node",
-    "author": "sambdavidson",
-    "author_url": "https://github.com/sambdavidson",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84534",
-    "pr_number": 84534,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "auth", "instrumentation", "node"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84543": {
-    "commit": "4d55d1d695225f1e4953feb999616451081b2f6f",
-    "text": "Update Azure SDK versions to v35.0.0",
-    "markdown": "Update Azure SDK versions to v35.0.0 ([#84543](https://github.com/kubernetes/kubernetes/pull/84543), [@andyzhangx](https://github.com/andyzhangx))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84543",
-    "pr_number": 84543,
-    "areas": ["cloudprovider", "dependency", "provider/azure"],
-    "kinds": ["bug", "cleanup", "feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84559": {
-    "commit": "ef479c1a6fdedca9fa396dbcc19f89a983ac47a9",
-    "text": "kubeadm now propagates proxy environment variables to kube-proxy",
-    "markdown": "kubeadm now propagates proxy environment variables to kube-proxy ([#84559](https://github.com/kubernetes/kubernetes/pull/84559), [@yastij](https://github.com/yastij))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "yastij",
-    "author_url": "https://github.com/yastij",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84559",
-    "pr_number": 84559,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84573": {
-    "commit": "ca836256c4234fd82b463ab03a857976141c6692",
-    "text": "Updated EndpointSlices to use PublishNotReadyAddresses from Services.",
-    "markdown": "Updated EndpointSlices to use PublishNotReadyAddresses from Services. ([#84573](https://github.com/kubernetes/kubernetes/pull/84573), [@robscott](https://github.com/robscott))\n\n  Courtesy of SIG Apps, and SIG Network",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84573",
-    "pr_number": 84573,
-    "kinds": ["feature"],
-    "sigs": ["apps", "network"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84591": {
-    "commit": "3c7706bf9c9dc1f0c2350b1f8c6d94e2536ec55e",
-    "text": "kubeadm: the command \"kubeadm token create\" now has a \"--certificate-key\" flag that can be used for the formation of join commands for control-planes with automatic copy of certificates",
-    "markdown": "kubeadm: the command \"kubeadm token create\" now has a \"--certificate-key\" flag that can be used for the formation of join commands for control-planes with automatic copy of certificates ([#84591](https://github.com/kubernetes/kubernetes/pull/84591), [@TheLastProject](https://github.com/TheLastProject))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "TheLastProject",
-    "author_url": "https://github.com/TheLastProject",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84591",
-    "pr_number": 84591,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84605": {
-    "commit": "1488460cd2c4948da5fa613655b73e92ff788c96",
-    "text": "add azure disk encryption(SSE+CMK) support",
-    "markdown": "add azure disk encryption(SSE+CMK) support ([#84605](https://github.com/kubernetes/kubernetes/pull/84605), [@andyzhangx](https://github.com/andyzhangx))\n\n  Courtesy of SIG Cloud Provider, and SIG Storage",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84605",
-    "pr_number": 84605,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug", "feature"],
-    "sigs": ["cloud-provider", "storage"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84614": {
-    "commit": "205570eb75b52bc0888309c71757314bd099b3d7",
-    "text": "kubelet now exports a \"server_expiration_renew_failure\" and \"client_expiration_renew_failure\" metric counter if the certificate rotations cannot be performed.",
-    "markdown": "kubelet now exports a \"server_expiration_renew_failure\" and \"client_expiration_renew_failure\" metric counter if the certificate rotations cannot be performed. ([#84614](https://github.com/kubernetes/kubernetes/pull/84614), [@rphillips](https://github.com/rphillips))\n\n  Courtesy of SIG API Machinery, SIG Auth, SIG CLI, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Instrumentation, SIG Node, and SIG Release",
-    "author": "rphillips",
-    "author_url": "https://github.com/rphillips",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84614",
-    "pr_number": 84614,
-    "areas": [
-      "apiserver",
-      "cloudprovider",
-      "code-generation",
-      "dependency",
-      "kubectl",
-      "kubelet",
-      "release-eng"
-    ],
-    "kinds": ["feature"],
-    "sigs": [
-      "api-machinery",
-      "auth",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "node",
-      "release"
-    ],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84622": {
-    "commit": "303de85dd188d16db808baba5857b7f0e6ecc1db",
-    "text": "Change GCP ILB firewall names to contain the \"k8s-fw-\" prefix like the rest of the firewall rules. This is needed for consistency and also for other components to identify the firewall rule as k8s/service-controller managed.",
-    "markdown": "Change GCP ILB firewall names to contain the \"k8s-fw-\" prefix like the rest of the firewall rules. This is needed for consistency and also for other components to identify the firewall rule as k8s/service-controller managed. ([#84622](https://github.com/kubernetes/kubernetes/pull/84622), [@prameshj](https://github.com/prameshj))",
-    "author": "prameshj",
-    "author_url": "https://github.com/prameshj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84622",
-    "pr_number": 84622,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.17.0"
-  },
-  "84637": {
-    "commit": "b55440dcf853aa517f991992bcb77eb1a708d59d",
-    "text": "People can see the right log and note.",
-    "markdown": "People can see the right log and note. ([#84637](https://github.com/kubernetes/kubernetes/pull/84637), [@zhipengzuo](https://github.com/zhipengzuo))",
-    "author": "zhipengzuo",
-    "author_url": "https://github.com/zhipengzuo",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84637",
-    "pr_number": 84637,
-    "kinds": ["cleanup"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84646": {
-    "commit": "077f825c584903eeb5854040192c2977079343e9",
-    "text": "Only validate duplication of the RequestedToCapacityRatio custom priority and allow other custom predicates/priorities",
-    "markdown": "Only validate duplication of the RequestedToCapacityRatio custom priority and allow other custom predicates/priorities ([#84646](https://github.com/kubernetes/kubernetes/pull/84646), [@liu-cong](https://github.com/liu-cong))",
-    "author": "liu-cong",
-    "author_url": "https://github.com/liu-cong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84646",
-    "pr_number": 84646,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.17.0"
-  },
-  "84648": {
-    "commit": "2f3376e4632a4433c98c49ccaa212216bb2b92bc",
-    "text": "kubeadm: prevent potential hanging of commands such as \"kubeadm reset\" if the apiserver endpoint is not reachable.",
-    "markdown": "kubeadm: prevent potential hanging of commands such as \"kubeadm reset\" if the apiserver endpoint is not reachable. ([#84648](https://github.com/kubernetes/kubernetes/pull/84648), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84648",
-    "pr_number": 84648,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "84662": {
-    "commit": "84fcf126c1624ea95174af0492f99339fc1a8850",
-    "text": "Simple script based hyperkube image that bundles all the necessary binaries. This is a equivalent replacement for the image based on the go based hyperkube command + image.",
-    "markdown": "Simple script based hyperkube image that bundles all the necessary binaries. This is a equivalent replacement for the image based on the go based hyperkube command + image. ([#84662](https://github.com/kubernetes/kubernetes/pull/84662), [@dims](https://github.com/dims))",
-    "author": "dims",
-    "author_url": "https://github.com/dims",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84662",
-    "pr_number": 84662,
-    "areas": ["hyperkube", "release-eng"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "release"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84677": {
-    "commit": "aaa57078969bbb72e9ebe45198edb0899bb1f7df",
-    "text": "The certificate signer no longer accepts ca.key passwords via the CFSSL_CA_PK_PASSWORD environment variable. This capability was not prompted by user request, never advertised, and recommended against in the security audit.",
-    "markdown": "The certificate signer no longer accepts ca.key passwords via the CFSSL_CA_PK_PASSWORD environment variable. This capability was not prompted by user request, never advertised, and recommended against in the security audit. ([#84677](https://github.com/kubernetes/kubernetes/pull/84677), [@mikedanese](https://github.com/mikedanese))",
-    "author": "mikedanese",
-    "author_url": "https://github.com/mikedanese",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84677",
-    "pr_number": 84677,
-    "areas": ["dependency", "kubectl", "kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "apps", "auth", "cli", "node"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84692": {
-    "commit": "c28921f248a8e6c923096154c6e87efcc188b9f0",
-    "text": "Clients can request protobuf and json and correctly negotiate with the server for JSON for CRD objects, allowing all client libraries to request protobuf if it is available.  If an error occurs negotiating a watch with the server, the error is immediately return by the client `Watch()` method instead of being sent as an `Error` event on the watch stream.",
-    "markdown": "Clients can request protobuf and json and correctly negotiate with the server for JSON for CRD objects, allowing all client libraries to request protobuf if it is available.  If an error occurs negotiating a watch with the server, the error is immediately return by the client `Watch()` method instead of being sent as an `Error` event on the watch stream. ([#84692](https://github.com/kubernetes/kubernetes/pull/84692), [@smarterclayton](https://github.com/smarterclayton))",
-    "author": "smarterclayton",
-    "author_url": "https://github.com/smarterclayton",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84692",
-    "pr_number": 84692,
-    "areas": ["apiserver", "kubectl", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth", "cli", "network", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84693": {
-    "commit": "7c7ae977d5ccc011dc2e1fcb7f06c3da2da09991",
-    "text": "kube-apiserver: fixed a bug that could cause a goroutine leak if the apiserver encountered an encoding error serving a watch to a websocket watcher",
-    "markdown": "kube-apiserver: fixed a bug that could cause a goroutine leak if the apiserver encountered an encoding error serving a watch to a websocket watcher ([#84693](https://github.com/kubernetes/kubernetes/pull/84693), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84693",
-    "pr_number": 84693,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "84713": {
-    "commit": "64be5ae5f178018d510d930457509f6b67d443c2",
-    "text": "CRDs defaulting is promoted to GA. Note: the feature gate CustomResourceDefaulting will be removed in 1.18.",
-    "markdown": "CRDs defaulting is promoted to GA. Note: the feature gate CustomResourceDefaulting will be removed in 1.18. ([#84713](https://github.com/kubernetes/kubernetes/pull/84713), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84713",
-    "pr_number": 84713,
-    "areas": ["custom-resources", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84722": {
-    "commit": "8436dde049fb055e2c58c85f3f1dec5c06c36399",
-    "text": "Implement the documented API semantics of list-type and map-type atomic to reject non-atomic sub-types.",
-    "markdown": "Implement the documented API semantics of list-type and map-type atomic to reject non-atomic sub-types. ([#84722](https://github.com/kubernetes/kubernetes/pull/84722), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84722",
-    "pr_number": 84722,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change", "bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "84727": {
-    "commit": "49a9b6cadfe6f27045e9ef05179b0af1cb6e9693",
-    "text": "If given an IPv6 bind-address, kube-apiserver will now advertise an IPv6 endpoint for the kubernetes.default service.",
-    "markdown": "If given an IPv6 bind-address, kube-apiserver will now advertise an IPv6 endpoint for the kubernetes.default service. ([#84727](https://github.com/kubernetes/kubernetes/pull/84727), [@danwinship](https://github.com/danwinship))",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84727",
-    "pr_number": 84727,
-    "areas": ["apiserver", "ipv6", "kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "network"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84754": {
-    "commit": "8548a253de80c787dbcd2b022faffc75c7c23025",
-    "text": "CSI Migration: Fixes issue where all volumes with the same inline volume inner spec name were staged in the same path. Migrated inline volumes are now staged at a unique path per unique volume.",
-    "markdown": "CSI Migration: Fixes issue where all volumes with the same inline volume inner spec name were staged in the same path. Migrated inline volumes are now staged at a unique path per unique volume. ([#84754](https://github.com/kubernetes/kubernetes/pull/84754), [@davidz627](https://github.com/davidz627))",
-    "author": "davidz627",
-    "author_url": "https://github.com/davidz627",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84754",
-    "pr_number": 84754,
-    "areas": ["dependency"],
-    "kinds": ["bug", "failing-test"],
-    "sigs": ["storage"],
-    "release_version": "1.17.0"
-  },
-  "84758": {
-    "commit": "7c2bfc0dda7c46a0a6e01e96bd6fa0e99813ed8f",
-    "text": "All resources within the rbac.authorization.k8s.io/v1alpha1 and rbac.authorization.k8s.io/v1beta1 API groups are deprecated in favor of rbac.authorization.k8s.io/v1, and will no longer be served in v1.20.",
-    "markdown": "All resources within the rbac.authorization.k8s.io/v1alpha1 and rbac.authorization.k8s.io/v1beta1 API groups are deprecated in favor of rbac.authorization.k8s.io/v1, and will no longer be served in v1.20. ([#84758](https://github.com/kubernetes/kubernetes/pull/84758), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84758",
-    "pr_number": 84758,
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["auth"],
-    "release_version": "1.17.0"
-  },
-  "84768": {
-    "commit": "681d22428b2e037163ea11c13a4c066c2058e515",
-    "text": "- kubelet and aggregated API servers now use v1 TokenReview and SubjectAccessReview endpoints to check authentication/authorization.\n- kube-apiserver can now specify `--authentication-token-webhook-version=v1` or `--authorization-webhook-version=v1` to use `v1` TokenReview and SubjectAccessReview API objects when communicating with authentication and authorization webhooks.",
-    "markdown": "- kubelet and aggregated API servers now use v1 TokenReview and SubjectAccessReview endpoints to check authentication/authorization.\n  - kube-apiserver can now specify `--authentication-token-webhook-version=v1` or `--authorization-webhook-version=v1` to use `v1` TokenReview and SubjectAccessReview API objects when communicating with authentication and authorization webhooks. ([#84768](https://github.com/kubernetes/kubernetes/pull/84768), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84768",
-    "pr_number": 84768,
-    "areas": ["apiserver", "kubelet", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "auth", "node", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84802": {
-    "commit": "e450a3d6f34f03b7d93d4d6378a9cbbd0864585c",
-    "text": "Ensure health probes are created for local traffic policy UDP services on Azure",
-    "markdown": "Ensure health probes are created for local traffic policy UDP services on Azure ([#84802](https://github.com/kubernetes/kubernetes/pull/84802), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84802",
-    "pr_number": 84802,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.17.0"
-  },
-  "84809": {
-    "commit": "41d9522a54b1faaadf1b1814777b626a98bf7eb4",
-    "text": "CSI Migration: GCE PD access mode now reflects read only status of inline volumes - this allows multi-attach for read only many PDs",
-    "markdown": "CSI Migration: GCE PD access mode now reflects read only status of inline volumes - this allows multi-attach for read only many PDs ([#84809](https://github.com/kubernetes/kubernetes/pull/84809), [@davidz627](https://github.com/davidz627))",
-    "author": "davidz627",
-    "author_url": "https://github.com/davidz627",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84809",
-    "pr_number": 84809,
-    "kinds": ["bug", "failing-test"],
-    "sigs": ["storage"],
-    "release_version": "1.17.0"
-  },
-  "84832": {
-    "commit": "72bcec4e4d516dcac227606541f990af13fb2918",
-    "text": "Bump CSI version to 1.2.0",
-    "markdown": "Bump CSI version to 1.2.0 ([#84832](https://github.com/kubernetes/kubernetes/pull/84832), [@gnufied](https://github.com/gnufied))\n\n  Courtesy of SIG Storage",
-    "author": "gnufied",
-    "author_url": "https://github.com/gnufied",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84832",
-    "pr_number": 84832,
-    "areas": ["dependency"],
-    "kinds": ["feature"],
-    "sigs": ["storage"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84835": {
-    "commit": "f23dd405f18f23dda4d00820ab20ec49852e5727",
-    "text": "Profiling is enabled by default in the scheduler",
-    "markdown": "Profiling is enabled by default in the scheduler ([#84835](https://github.com/kubernetes/kubernetes/pull/84835), [@denkensk](https://github.com/denkensk))\n\n  Courtesy of SIG Scheduling",
-    "author": "denkensk",
-    "author_url": "https://github.com/denkensk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84835",
-    "pr_number": 84835,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84882": {
-    "commit": "3eff237357e2df6a64cc14a5dc13c090edafb8be",
-    "text": "Moving WindowsRunAsUserName feature to beta",
-    "markdown": "Moving WindowsRunAsUserName feature to beta ([#84882](https://github.com/kubernetes/kubernetes/pull/84882), [@marosset](https://github.com/marosset))",
-    "author": "marosset",
-    "author_url": "https://github.com/marosset",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84882",
-    "pr_number": 84882,
-    "areas": ["kubelet"],
-    "kinds": ["api-change"],
-    "sigs": ["apps", "node", "windows"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84888": {
-    "commit": "c843d9614c93cf0867e72fba01cbd532d6579361",
-    "text": "kubeadm: fix an issue with the kube-proxy container env. variables",
-    "markdown": "kubeadm: fix an issue with the kube-proxy container env. variables ([#84888](https://github.com/kubernetes/kubernetes/pull/84888), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84888",
-    "pr_number": 84888,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "84896": {
-    "commit": "d366d2eaa36e0d27fca529ea0be6648f56513fda",
-    "text": "Following metrics from kube-controller-manager are now marked as with the ALPHA stability level:\nstorage_count_attachable_volumes_in_use\nattachdetach_controller_total_volumes\npv_collector_bound_pv_count\npv_collector_unbound_pv_count\npv_collector_bound_pvc_count\npv_collector_unbound_pvc_count",
-    "markdown": "Following metrics from kube-controller-manager are now marked as with the ALPHA stability level:\n  storage_count_attachable_volumes_in_use\n  attachdetach_controller_total_volumes\n  pv_collector_bound_pv_count\n  pv_collector_unbound_pv_count\n  pv_collector_bound_pvc_count\n  pv_collector_unbound_pvc_count ([#84896](https://github.com/kubernetes/kubernetes/pull/84896), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Apps",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84896",
-    "pr_number": 84896,
-    "kinds": ["feature"],
-    "sigs": ["apps"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84907": {
-    "commit": "b3dde20411901a9a6114381811623083e73ed440",
-    "text": "Following metrics from kubelet are now marked as with the ALPHA stability level:\nkubelet_container_log_filesystem_used_bytes\nkubelet_volume_stats_capacity_bytes\nkubelet_volume_stats_available_bytes\nkubelet_volume_stats_used_bytes\nkubelet_volume_stats_inodes\nkubelet_volume_stats_inodes_free\nkubelet_volume_stats_inodes_used\nplugin_manager_total_plugins\nvolume_manager_total_volumes",
-    "markdown": "Following metrics from kubelet are now marked as with the ALPHA stability level:\n  kubelet_container_log_filesystem_used_bytes\n  kubelet_volume_stats_capacity_bytes\n  kubelet_volume_stats_available_bytes\n  kubelet_volume_stats_used_bytes\n  kubelet_volume_stats_inodes\n  kubelet_volume_stats_inodes_free\n  kubelet_volume_stats_inodes_used\n  plugin_manager_total_plugins\n  volume_manager_total_volumes ([#84907](https://github.com/kubernetes/kubernetes/pull/84907), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Node",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84907",
-    "pr_number": 84907,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84917": {
-    "commit": "91a53b6869811f6b0338464779eb634da4ef1fcc",
-    "text": "fix race condition when attach/delete azure disk in same time",
-    "markdown": "fix race condition when attach/delete azure disk in same time ([#84917](https://github.com/kubernetes/kubernetes/pull/84917), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84917",
-    "pr_number": 84917,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.17.0"
-  },
-  "84957": {
-    "commit": "35de7a2537f73de74fa22288c09fe927d9cb21d6",
-    "text": "matches := stackCreator.FindStringSubmatch(string(stack))\nmatches type is []string\n\nso nil len(matches) ==0 \n\nchange:\n if matches == nil || len(matches) != 4 ==\u003eif len(matches) != 4",
-    "markdown": "matches := stackCreator.FindStringSubmatch(string(stack))\n  matches type is []string\n  \n  so nil len(matches) ==0 \n  \n  change:\n   if matches == nil || len(matches) != 4 ==\u003eif len(matches) != 4 ([#84957](https://github.com/kubernetes/kubernetes/pull/84957), [@ZP-AlwaysWin](https://github.com/ZP-AlwaysWin))",
-    "author": "ZP-AlwaysWin",
-    "author_url": "https://github.com/ZP-AlwaysWin",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84957",
-    "pr_number": 84957,
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84963": {
-    "commit": "15f586a6c18d7ccbc01f7a1e908dbabcfd146ae2",
-    "text": "kube-apiserver: Fixed a regression accepting patch requests \u003e 1MB",
-    "markdown": "kube-apiserver: Fixed a regression accepting patch requests \u003e 1MB ([#84963](https://github.com/kubernetes/kubernetes/pull/84963), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84963",
-    "pr_number": 84963,
-    "areas": ["apiserver", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84971": {
-    "commit": "c5609071d805036b251f22ce122bbf13ca94bff7",
-    "text": "Splitting IP address type into IPv4 and IPv6 for EndpointSlices",
-    "markdown": "Splitting IP address type into IPv4 and IPv6 for EndpointSlices ([#84971](https://github.com/kubernetes/kubernetes/pull/84971), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84971",
-    "pr_number": 84971,
-    "areas": ["dependency", "ipvs", "kubectl", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["apps", "cli", "network", "scheduling", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "84987": {
-    "commit": "ec86baf00ba6478362031f32787e69765da4bfeb",
-    "text": "Following metrics from kubelet are now marked as with the ALPHA stability level:\nnode_cpu_usage_seconds_total\nnode_memory_working_set_bytes\ncontainer_cpu_usage_seconds_total\ncontainer_memory_working_set_bytes\nscrape_error",
-    "markdown": "Following metrics from kubelet are now marked as with the ALPHA stability level:\n  node_cpu_usage_seconds_total\n  node_memory_working_set_bytes\n  container_cpu_usage_seconds_total\n  container_memory_working_set_bytes\n  scrape_error ([#84987](https://github.com/kubernetes/kubernetes/pull/84987), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG API Machinery, SIG Cluster Lifecycle, SIG Instrumentation, and SIG Node",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84987",
-    "pr_number": 84987,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "instrumentation", "node"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "84998": {
-    "commit": "43ccfced1dc1db6fcf7209e710021518b3d42ae2",
-    "text": "Fix a bug that a node Lease object may have been created without OwnerReference.",
-    "markdown": "Fix a bug that a node Lease object may have been created without OwnerReference. ([#84998](https://github.com/kubernetes/kubernetes/pull/84998), [@wojtek-t](https://github.com/wojtek-t))",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84998",
-    "pr_number": 84998,
-    "areas": ["kubelet", "test"],
-    "kinds": ["bug"],
-    "sigs": ["node", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85004": {
-    "commit": "02af1dd62c4842e20e2ee7337edf032327b1c8ed",
-    "text": "apiservers based on k8s.io/apiserver with delegated authn based on cluster authentication will automatically update to new authentication information when the authoritative configmap is updated.",
-    "markdown": "apiservers based on k8s.io/apiserver with delegated authn based on cluster authentication will automatically update to new authentication information when the authoritative configmap is updated. ([#85004](https://github.com/kubernetes/kubernetes/pull/85004), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85004",
-    "pr_number": 85004,
-    "areas": ["apiserver", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85023": {
-    "commit": "072cf5bd58b3699701d8f5d85241442c872c7c8b",
-    "text": "Finalizer Protection for Service LoadBalancers is now in GA (enabled by default). This feature ensures the Service resource is not fully deleted until the correlating load balancer resources are deleted.",
-    "markdown": "Finalizer Protection for Service LoadBalancers is now in GA (enabled by default). This feature ensures the Service resource is not fully deleted until the correlating load balancer resources are deleted. ([#85023](https://github.com/kubernetes/kubernetes/pull/85023), [@MrHohn](https://github.com/MrHohn))\n\n  Courtesy of SIG Apps, and SIG Network",
-    "author": "MrHohn",
-    "author_url": "https://github.com/MrHohn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85023",
-    "pr_number": 85023,
-    "kinds": ["feature"],
-    "sigs": ["apps", "network"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "85024": {
-    "commit": "9920b733ba56f908d48dc7d0a30bc90d28d015a4",
-    "text": "kubeadm: fix skipped etcd upgrade on secondary control-plane nodes when the command \"kubeadm upgrade node\" is used.",
-    "markdown": "kubeadm: fix skipped etcd upgrade on secondary control-plane nodes when the command \"kubeadm upgrade node\" is used. ([#85024](https://github.com/kubernetes/kubernetes/pull/85024), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85024",
-    "pr_number": 85024,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "85043": {
-    "commit": "939e1e64881c0fc920b75c7489816fbf72806358",
-    "text": "kubeadm: enable the usage of the secure kube-scheduler and kube-controller-manager ports for health checks. For kube-scheduler was 10251, becomes 10259. For kube-controller-manager was 10252, becomes 10257.",
-    "markdown": "kubeadm: enable the usage of the secure kube-scheduler and kube-controller-manager ports for health checks. For kube-scheduler was 10251, becomes 10259. For kube-controller-manager was 10252, becomes 10257. ([#85043](https://github.com/kubernetes/kubernetes/pull/85043), [@neolit123](https://github.com/neolit123))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85043",
-    "pr_number": 85043,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "85044": {
-    "commit": "fc0bf06983e32424cc90bff37db561a7dde95943",
-    "text": "kubeadm: remove the deprecated \"--cri-socket\" flag for \"kubeadm upgrade apply\". The flag has been deprecated since v1.14.",
-    "markdown": "kubeadm: remove the deprecated \"--cri-socket\" flag for \"kubeadm upgrade apply\". The flag has been deprecated since v1.14. ([#85044](https://github.com/kubernetes/kubernetes/pull/85044), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85044",
-    "pr_number": 85044,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "85048": {
-    "commit": "be1658e08c6762d89c7f1fdc0655bb0a45dad1fb",
-    "text": "add RequiresExactMatch for label.Selector",
-    "markdown": "add RequiresExactMatch for label.Selector ([#85048](https://github.com/kubernetes/kubernetes/pull/85048), [@shaloulcy](https://github.com/shaloulcy))\n\n  Courtesy of SIG API Machinery",
-    "author": "shaloulcy",
-    "author_url": "https://github.com/shaloulcy",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85048",
-    "pr_number": 85048,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "85091": {
-    "commit": "4f6aa5a921f9b10ea7dce399f5b3c360953528bb",
-    "text": "Fixed bug when using kubeadm alpha certs commands with clusters using external etcd",
-    "markdown": "Fixed bug when using kubeadm alpha certs commands with clusters using external etcd ([#85091](https://github.com/kubernetes/kubernetes/pull/85091), [@fabriziopandini](https://github.com/fabriziopandini))",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85091",
-    "pr_number": 85091,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "85092": {
-    "commit": "e008523f76497a31a0bf62602357f5117c5d5f54",
-    "text": "kubeadm alpha certs command now skip missing files",
-    "markdown": "kubeadm alpha certs command now skip missing files ([#85092](https://github.com/kubernetes/kubernetes/pull/85092), [@fabriziopandini](https://github.com/fabriziopandini))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85092",
-    "pr_number": 85092,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "85094": {
-    "commit": "0968636760dae228581b7b515cf605fccd7e2969",
-    "text": "kubeadm deprecates the use of the hyperkube image",
-    "markdown": "kubeadm deprecates the use of the hyperkube image ([#85094](https://github.com/kubernetes/kubernetes/pull/85094), [@rosti](https://github.com/rosti))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85094",
-    "pr_number": 85094,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "action_required": true,
-    "release_version": "1.17.0"
-  },
-  "85098": {
-    "commit": "b49afbfa64b6be72779b6deb424d7f0cddd5ccf1",
-    "text": "kube-apiserver: The `AdmissionConfiguration` type accepted by `--admission-control-config-file` has been promoted to `apiserver.config.k8s.io/v1` with no schema changes.",
-    "markdown": "kube-apiserver: The `AdmissionConfiguration` type accepted by `--admission-control-config-file` has been promoted to `apiserver.config.k8s.io/v1` with no schema changes. ([#85098](https://github.com/kubernetes/kubernetes/pull/85098), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85098",
-    "pr_number": 85098,
-    "areas": ["apiserver"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "85099": {
-    "commit": "195664db0eb633903f0098bd07ded2898aa40e9a",
-    "text": "kube-apiserver: The `ResourceQuota` admission plugin configuration referenced from `--admission-control-config-file` admission config has been promoted to `apiVersion: apiserver.config.k8s.io/v1`, `kind: ResourceQuotaConfiguration` with no schema changes.",
-    "markdown": "kube-apiserver: The `ResourceQuota` admission plugin configuration referenced from `--admission-control-config-file` admission config has been promoted to `apiVersion: apiserver.config.k8s.io/v1`, `kind: ResourceQuotaConfiguration` with no schema changes. ([#85099](https://github.com/kubernetes/kubernetes/pull/85099), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85099",
-    "pr_number": 85099,
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "85109": {
-    "commit": "d0f021524e908c8b646563eb1864a0d3782db963",
-    "text": "Kubeadm now includes CoreDNS version 1.6.5\n - `kubernetes` plugin adds metrics to measure kubernetes control plane latency.\n -  the `health` plugin now includes the `lameduck` option by default, which waits for a duration before shutting down.",
-    "markdown": "Kubeadm now includes CoreDNS version 1.6.5\n   - `kubernetes` plugin adds metrics to measure kubernetes control plane latency.\n   -  the `health` plugin now includes the `lameduck` option by default, which waits for a duration before shutting down. ([#85109](https://github.com/kubernetes/kubernetes/pull/85109), [@rajansandeep](https://github.com/rajansandeep))",
-    "author": "rajansandeep",
-    "author_url": "https://github.com/rajansandeep",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85109",
-    "pr_number": 85109,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "85115": {
-    "commit": "5dd641e45cc29957db2f75d64a41c782b08a953d",
-    "text": "azure: update disk lock logic per vm during attach/detach to allow concurrent updates for different nodes.",
-    "markdown": "azure: update disk lock logic per vm during attach/detach to allow concurrent updates for different nodes. ([#85115](https://github.com/kubernetes/kubernetes/pull/85115), [@aramase](https://github.com/aramase))",
-    "author": "aramase",
-    "author_url": "https://github.com/aramase",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85115",
-    "pr_number": 85115,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.17.0"
-  },
-  "85138": {
-    "commit": "f501d8e59a599eaad45a50ee1da075cedc9d0ab9",
-    "text": "kube-apiserver: Authentication configuration for mutating and validating admission webhooks referenced from an `--admission-control-config-file` can now be specified with `apiVersion: apiserver.config.k8s.io/v1, kind: WebhookAdmissionConfiguration`.",
-    "markdown": "kube-apiserver: Authentication configuration for mutating and validating admission webhooks referenced from an `--admission-control-config-file` can now be specified with `apiVersion: apiserver.config.k8s.io/v1, kind: WebhookAdmissionConfiguration`. ([#85138](https://github.com/kubernetes/kubernetes/pull/85138), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85138",
-    "pr_number": 85138,
-    "areas": ["apiserver"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "85156": {
-    "commit": "f5df681b809f774056d934294f87858909b8a781",
-    "text": "Fixes a bug in kubeadm that caused init and join to hang indefinitely in specific conditions.",
-    "markdown": "Fixes a bug in kubeadm that caused init and join to hang indefinitely in specific conditions. ([#85156](https://github.com/kubernetes/kubernetes/pull/85156), [@chuckha](https://github.com/chuckha))",
-    "author": "chuckha",
-    "author_url": "https://github.com/chuckha",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85156",
-    "pr_number": 85156,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "85158": {
-    "commit": "1057f3a82ac4460ca40753614962c39f238913b6",
-    "text": "fix vmss dirty cache issue in disk attach/detach on vmss node",
-    "markdown": "fix vmss dirty cache issue in disk attach/detach on vmss node ([#85158](https://github.com/kubernetes/kubernetes/pull/85158), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85158",
-    "pr_number": 85158,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.17.0"
-  },
-  "85174": {
-    "commit": "547fdcc1640f614c21d8cc1040c7f9579c68c0c0",
-    "text": "Add table convertor to componentstatus.",
-    "markdown": "Add table convertor to componentstatus. ([#85174](https://github.com/kubernetes/kubernetes/pull/85174), [@zhouya0](https://github.com/zhouya0))",
-    "author": "zhouya0",
-    "author_url": "https://github.com/zhouya0",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85174",
-    "pr_number": 85174,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cli"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85201": {
-    "commit": "d1e8702d36947ccc23ad820eaae7f4afbaf0b058",
-    "text": "kubeadm: added retry to all the calls to the etcd API so kubeadm will be more resilient to network glitches",
-    "markdown": "kubeadm: added retry to all the calls to the etcd API so kubeadm will be more resilient to network glitches ([#85201](https://github.com/kubernetes/kubernetes/pull/85201), [@fabriziopandini](https://github.com/fabriziopandini))",
-    "author": "fabriziopandini",
-    "author_url": "https://github.com/fabriziopandini",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85201",
-    "pr_number": 85201,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "85223": {
-    "commit": "976712556e4bd22d5312a0af36b18127c709d54a",
-    "text": "CRDs can have fields named `type` with value `array` and nested array with `items` fields without validation to fall over this.",
-    "markdown": "CRDs can have fields named `type` with value `array` and nested array with `items` fields without validation to fall over this. ([#85223](https://github.com/kubernetes/kubernetes/pull/85223), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85223",
-    "pr_number": 85223,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl"],
-    "kinds": ["bug"],
-    "sigs": [
-      "api-machinery",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "node"
-    ],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85228": {
-    "commit": "02c61b6af1250bfabf99628870cfa689bd790c62",
-    "text": "Wait for kubelet \u0026 kube-proxy to be ready on Windows node within 10s",
-    "markdown": "Wait for kubelet \u0026 kube-proxy to be ready on Windows node within 10s ([#85228](https://github.com/kubernetes/kubernetes/pull/85228), [@YangLu1031](https://github.com/YangLu1031))",
-    "author": "YangLu1031",
-    "author_url": "https://github.com/YangLu1031",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85228",
-    "pr_number": 85228,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "85231": {
-    "commit": "9c26a752a9cfa45423ba2d8148b16b22de31e65b",
-    "text": "Feature gates CSIMigration to Beta (on by default) and CSIMigrationGCE to Beta (off by default since it requires installation of the GCE PD CSI Driver)\nThe in-tree GCE PD plugin \"kubernetes.io/gce-pd\" is now deprecated and will be removed in 1.21. Users should enable CSIMigration + CSIMigrationGCE features and install the GCE PD CSI Driver (https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) to avoid disruption to existing Pod and PVC objects at that time.\nUsers should start using the GCE PD CSI CSI Driver directly for any new volumes.",
-    "markdown": "Feature gates CSIMigration to Beta (on by default) and CSIMigrationGCE to Beta (off by default since it requires installation of the GCE PD CSI Driver)\n  The in-tree GCE PD plugin \"kubernetes.io/gce-pd\" is now deprecated and will be removed in 1.21. Users should enable CSIMigration + CSIMigrationGCE features and install the GCE PD CSI Driver (https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) to avoid disruption to existing Pod and PVC objects at that time.\n  Users should start using the GCE PD CSI CSI Driver directly for any new volumes. ([#85231](https://github.com/kubernetes/kubernetes/pull/85231), [@davidz627](https://github.com/davidz627))\n\n  Courtesy of SIG Apps, SIG Auth, SIG Node, SIG Storage, and SIG Testing",
-    "author": "davidz627",
-    "author_url": "https://github.com/davidz627",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85231",
-    "pr_number": 85231,
-    "areas": ["kubelet", "test"],
-    "kinds": ["documentation", "feature"],
-    "sigs": ["apps", "auth", "node", "storage", "testing"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "85237": {
-    "commit": "81af5baff8e958dc8817913ec3c4172028f30122",
-    "text": "Promote CSIMigrationAWS to Beta (off by default since it requires installation of the AWS EBS CSI Driver)\nThe in-tree AWS EBS plugin \"kubernetes.io/aws-ebs\" is now deprecated and will be removed in 1.21. Users should enable CSIMigration + CSIMigrationAWS features and install the AWS EBS CSI Driver (https://github.com/kubernetes-sigs/aws-ebs-csi-driver) to avoid disruption to existing Pod and PVC objects at that time.\nUsers should start using the AWS EBS CSI CSI Driver directly for any new volumes.",
-    "markdown": "Promote CSIMigrationAWS to Beta (off by default since it requires installation of the AWS EBS CSI Driver)\n  The in-tree AWS EBS plugin \"kubernetes.io/aws-ebs\" is now deprecated and will be removed in 1.21. Users should enable CSIMigration + CSIMigrationAWS features and install the AWS EBS CSI Driver (https://github.com/kubernetes-sigs/aws-ebs-csi-driver) to avoid disruption to existing Pod and PVC objects at that time.\n  Users should start using the AWS EBS CSI CSI Driver directly for any new volumes. ([#85237](https://github.com/kubernetes/kubernetes/pull/85237), [@leakingtapan](https://github.com/leakingtapan))\n\n  Courtesy of SIG Storage",
-    "author": "leakingtapan",
-    "author_url": "https://github.com/leakingtapan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85237",
-    "pr_number": 85237,
-    "areas": ["provider/aws"],
-    "kinds": ["feature"],
-    "sigs": ["storage"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "85246": {
-    "commit": "27067540ffc97495e7a63f402b4a2fda01f40085",
-    "text": "kube-proxy now supports DualStack feature with EndpointSlices and IPVS.",
-    "markdown": "kube-proxy now supports DualStack feature with EndpointSlices and IPVS. ([#85246](https://github.com/kubernetes/kubernetes/pull/85246), [@robscott](https://github.com/robscott))\n\n  Courtesy of SIG Network",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85246",
-    "pr_number": 85246,
-    "areas": ["ipvs"],
-    "kinds": ["feature"],
-    "sigs": ["network"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "85260": {
-    "commit": "9d0a32caf8bf9f2fdc31ee586938bdbaba64fab4",
-    "text": "Renamed FeatureGate RequestManagement to APIPriorityAndFairness.  This feature gate is an alpha and has not yet been associated with any actual functionality.\nchange references to feature gate RequestManagement into references to APIPriorityAndFairness",
-    "markdown": "Renamed FeatureGate RequestManagement to APIPriorityAndFairness.  This feature gate is an alpha and has not yet been associated with any actual functionality.\n  change references to feature gate RequestManagement into references to APIPriorityAndFairness ([#85260](https://github.com/kubernetes/kubernetes/pull/85260), [@MikeSpreitzer](https://github.com/MikeSpreitzer))\n\n  Courtesy of SIG API Machinery",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190228-priority-and-fairness.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "MikeSpreitzer",
-    "author_url": "https://github.com/MikeSpreitzer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85260",
-    "pr_number": 85260,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "action_required": true,
-    "release_version": "1.17.0"
-  },
-  "85265": {
-    "commit": "f01a4d06085b7b44844c472d6dedbd0538080c78",
-    "text": "kubeadm: reset raises warnings if it cannot delete folders",
-    "markdown": "kubeadm: reset raises warnings if it cannot delete folders ([#85265](https://github.com/kubernetes/kubernetes/pull/85265), [@SataQiu](https://github.com/SataQiu))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85265",
-    "pr_number": 85265,
-    "areas": ["kubeadm"],
-    "kinds": ["bug", "feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "85285": {
-    "commit": "141329fd2157a0791b93fae0160ce24d99808607",
-    "text": "kubectl: --resource-version now works properly in label/annotate/set selector commands when racing with other clients to update the target object",
-    "markdown": "kubectl: --resource-version now works properly in label/annotate/set selector commands when racing with other clients to update the target object ([#85285](https://github.com/kubernetes/kubernetes/pull/85285), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85285",
-    "pr_number": 85285,
-    "areas": ["kubectl", "test"],
-    "kinds": ["bug"],
-    "sigs": ["cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85293": {
-    "commit": "2b6857db4d4226cc07180308fe8a8507abbc7700",
-    "text": "Includes FSType when describing CSI persistent volumes.",
-    "markdown": "Includes FSType when describing CSI persistent volumes. ([#85293](https://github.com/kubernetes/kubernetes/pull/85293), [@huffmanca](https://github.com/huffmanca))",
-    "author": "huffmanca",
-    "author_url": "https://github.com/huffmanca",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85293",
-    "pr_number": 85293,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli", "storage"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85346": {
-    "commit": "f16bfb069a22241a5501f6fe530f5d4e2a82cf0e",
-    "text": "Remove redundant API validation when using Service Topology with externalTrafficPolicy=Local",
-    "markdown": "Remove redundant API validation when using Service Topology with externalTrafficPolicy=Local ([#85346](https://github.com/kubernetes/kubernetes/pull/85346), [@andrewsykim](https://github.com/andrewsykim))",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85346",
-    "pr_number": 85346,
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["network"],
-    "release_version": "1.17.0"
-  },
-  "85351": {
-    "commit": "01e1dcf429cf75a6fed447fc4ff30b8a17df9c83",
-    "text": "kubeadm: Fix a bug where kubeadm cannot parse kubelet's version if the latter dumps logs on the standard error.",
-    "markdown": "kubeadm: Fix a bug where kubeadm cannot parse kubelet's version if the latter dumps logs on the standard error. ([#85351](https://github.com/kubernetes/kubernetes/pull/85351), [@rosti](https://github.com/rosti))",
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85351",
-    "pr_number": 85351,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "85359": {
-    "commit": "f2469a7a2c75a78b8c0b0514652511d105d2b2e7",
-    "text": "When upgrading to 1.17 with a cluster with EndpointSlices enabled, the `endpointslice.kubernetes.io/managed-by` label needs to be set on each EndpointSlice.",
-    "markdown": "When upgrading to 1.17 with a cluster with EndpointSlices enabled, the `endpointslice.kubernetes.io/managed-by` label needs to be set on each EndpointSlice. ([#85359](https://github.com/kubernetes/kubernetes/pull/85359), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85359",
-    "pr_number": 85359,
-    "areas": ["test"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "network", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85361": {
-    "commit": "9859e73433095b8f05695c6ec450215d173cf27a",
-    "text": "kube-controller-manager: Fixes bug setting headless service labels on endpoints",
-    "markdown": "kube-controller-manager: Fixes bug setting headless service labels on endpoints ([#85361](https://github.com/kubernetes/kubernetes/pull/85361), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85361",
-    "pr_number": 85361,
-    "kinds": ["bug"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85363": {
-    "commit": "0810bc33869b9b2e5bbabef4fa41e99d53526cdb",
-    "text": "1. Currently, if users were to explicitly specify CacheSize of 0 for KMS provider, they would end-up with a provider that caches up to 1000 keys. This PR changes this behavior.\nPost this PR, when users supply 0 for CacheSize this will result in a validation error.\n2. CacheSize type was changed from int32 to *int32. This allows defaulting logic to differentiate between cases where users explicitly supplied 0 vs. not supplied any value.\n3. KMS Provider's endpoint (path to Unix socket) is now validated when the EncryptionConfiguration files is loaded. This used to be handled by the GRPCService.",
-    "markdown": "1. Currently, if users were to explicitly specify CacheSize of 0 for KMS provider, they would end-up with a provider that caches up to 1000 keys. This PR changes this behavior.\n  Post this PR, when users supply 0 for CacheSize this will result in a validation error.\n  2. CacheSize type was changed from int32 to *int32. This allows defaulting logic to differentiate between cases where users explicitly supplied 0 vs. not supplied any value.\n  3. KMS Provider's endpoint (path to Unix socket) is now validated when the EncryptionConfiguration files is loaded. This used to be handled by the GRPCService. ([#85363](https://github.com/kubernetes/kubernetes/pull/85363), [@immutableT](https://github.com/immutableT))",
-    "author": "immutableT",
-    "author_url": "https://github.com/immutableT",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85363",
-    "pr_number": 85363,
-    "areas": ["apiserver"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "auth", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85365": {
-    "commit": "5848ee494593dea2556a81c0f7a69ac3f7754d1a",
-    "text": "EndpointSlices are not enabled by default. Use the EndpointSlice feature gate to enable this feature.",
-    "markdown": "EndpointSlices are not enabled by default. Use the EndpointSlice feature gate to enable this feature. ([#85365](https://github.com/kubernetes/kubernetes/pull/85365), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85365",
-    "pr_number": 85365,
-    "kinds": ["bug"],
-    "sigs": ["auth", "network"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85368": {
-    "commit": "9905a33fd738a52d3f7ae73cf823d82a05720e33",
-    "text": "Fix bug where EndpointSlice controller would attempt to modify shared objects.",
-    "markdown": "Fix bug where EndpointSlice controller would attempt to modify shared objects. ([#85368](https://github.com/kubernetes/kubernetes/pull/85368), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85368",
-    "pr_number": 85368,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "apps", "network"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85381": {
-    "commit": "ac2a1b7837d0d38cbe57a9d51387737bfb7a6a32",
-    "text": "OpenAPI v3 format in CustomResourceDefinition schemas are now documented.",
-    "markdown": "OpenAPI v3 format in CustomResourceDefinition schemas are now documented. ([#85381](https://github.com/kubernetes/kubernetes/pull/85381), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85381",
-    "pr_number": 85381,
-    "areas": ["test"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85408": {
-    "commit": "bf8c276019331f589dbdd2007e552a98d2e50eed",
-    "text": "Use context to check client closed instead of http.CloseNotifier in processing watch request which will reduce 1 goroutine for each request if proto is HTTP/2.x .",
-    "markdown": "Use context to check client closed instead of http.CloseNotifier in processing watch request which will reduce 1 goroutine for each request if proto is HTTP/2.x . ([#85408](https://github.com/kubernetes/kubernetes/pull/85408), [@answer1991](https://github.com/answer1991))",
-    "author": "answer1991",
-    "author_url": "https://github.com/answer1991",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85408",
-    "pr_number": 85408,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "85412": {
-    "commit": "c65c56348760b5a3c10d9ec214f99ddb3add1488",
-    "text": "kubeadm: don't write the kubelet environment file on \"upgrade apply\"",
-    "markdown": "kubeadm: don't write the kubelet environment file on \"upgrade apply\" ([#85412](https://github.com/kubernetes/kubernetes/pull/85412), [@boluisa](https://github.com/boluisa))",
-    "author": "boluisa",
-    "author_url": "https://github.com/boluisa",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85412",
-    "pr_number": 85412,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "85441": {
-    "commit": "384e45febdb302c145ab0e1fd3bcf0eda882e2e8",
-    "text": "Resolves error from v1.17.0-beta.2 with --authorizer-mode webhook complaining about an invalid version",
-    "markdown": "Resolves error from v1.17.0-beta.2 with --authorizer-mode webhook complaining about an invalid version ([#85441](https://github.com/kubernetes/kubernetes/pull/85441), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85441",
-    "pr_number": 85441,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85444": {
-    "commit": "5d3125e51258646fff19089f9eb2f2f80c784385",
-    "text": "kube-apiserver: Fixes a bug that hidden metrics can not be enabled by the command-line option `--show-hidden-metrics-for-version`.",
-    "markdown": "kube-apiserver: Fixes a bug that hidden metrics can not be enabled by the command-line option `--show-hidden-metrics-for-version`. ([#85444](https://github.com/kubernetes/kubernetes/pull/85444), [@RainbowMango](https://github.com/RainbowMango))",
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85444",
-    "pr_number": 85444,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85466": {
-    "commit": "d9ed6110e88e73da597cc03a38ee1357f9b67ed3",
-    "text": "Windows nodes on GCE can use TPM-based authentication to the master.",
-    "markdown": "Windows nodes on GCE can use TPM-based authentication to the master. ([#85466](https://github.com/kubernetes/kubernetes/pull/85466), [@pjh](https://github.com/pjh))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "pjh",
-    "author_url": "https://github.com/pjh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85466",
-    "pr_number": 85466,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "85475": {
-    "commit": "97e34ee223c1a7e2c83399d789c8139d29be88aa",
-    "text": "fix azure file AuthorizationFailure",
-    "markdown": "fix azure file AuthorizationFailure ([#85475](https://github.com/kubernetes/kubernetes/pull/85475), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85475",
-    "pr_number": 85475,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85490": {
-    "commit": "7c87b5fb55ca096c007c8739d4657a5a4e29fb09",
-    "text": "kubectl set resources will no longer return an error if passed an empty change for a resource. \nkubectl set subject will no longer return an error if passed an empty change for a resource.",
-    "markdown": "kubectl set resources will no longer return an error if passed an empty change for a resource. \n  kubectl set subject will no longer return an error if passed an empty change for a resource. ([#85490](https://github.com/kubernetes/kubernetes/pull/85490), [@sallyom](https://github.com/sallyom))",
-    "author": "sallyom",
-    "author_url": "https://github.com/sallyom",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85490",
-    "pr_number": 85490,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.17.0"
-  },
-  "85494": {
-    "commit": "3c5dad61f72cc0fcee20bc5c27db1bde31a691b9",
-    "text": "kubeadm: fix stray \"node-cidr-mask-size\" flag in the kube-controller-manager manifest when IPv6DualStack is enabled",
-    "markdown": "kubeadm: fix stray \"node-cidr-mask-size\" flag in the kube-controller-manager manifest when IPv6DualStack is enabled ([#85494](https://github.com/kubernetes/kubernetes/pull/85494), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85494",
-    "pr_number": 85494,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "85516": {
-    "commit": "1682a4a0b91edc64b42728a2b92aefe48d69ccdb",
-    "text": "kube-apiserver: fixed a conflict error encountered attempting to delete a pod with gracePeriodSeconds=0 and a resourceVersion precondition",
-    "markdown": "kube-apiserver: fixed a conflict error encountered attempting to delete a pod with gracePeriodSeconds=0 and a resourceVersion precondition ([#85516](https://github.com/kubernetes/kubernetes/pull/85516), [@michaelgugino](https://github.com/michaelgugino))",
-    "author": "michaelgugino",
-    "author_url": "https://github.com/michaelgugino",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85516",
-    "pr_number": 85516,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "85524": {
-    "commit": "3028dc1f7ac6dfd87463e6d21466966984990bf5",
-    "text": "kubeadm: fix a panic in case the KubeProxyConfiguration feature gates were not initialized.",
-    "markdown": "kubeadm: fix a panic in case the KubeProxyConfiguration feature gates were not initialized. ([#85524](https://github.com/kubernetes/kubernetes/pull/85524), [@Arvinderpal](https://github.com/Arvinderpal))",
-    "author": "Arvinderpal",
-    "author_url": "https://github.com/Arvinderpal",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85524",
-    "pr_number": 85524,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "85543": {
-    "commit": "650c797d3faedcbd3ad60cad065cabf305bb49a1",
-    "text": "addons: elasticsearch discovery supports IPv6",
-    "markdown": "addons: elasticsearch discovery supports IPv6 ([#85543](https://github.com/kubernetes/kubernetes/pull/85543), [@SataQiu](https://github.com/SataQiu))",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85543",
-    "pr_number": 85543,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85571": {
-    "commit": "b42f21982c965630f82f053425ba067d818203f5",
-    "text": "kubectl/drain: add disable-eviction option.\nForce drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, and should be used with caution.",
-    "markdown": "kubectl/drain: add disable-eviction option.\n  Force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, and should be used with caution. ([#85571](https://github.com/kubernetes/kubernetes/pull/85571), [@michaelgugino](https://github.com/michaelgugino))\n\n  Courtesy of SIG CLI",
-    "author": "michaelgugino",
-    "author_url": "https://github.com/michaelgugino",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85571",
-    "pr_number": 85571,
-    "areas": ["kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "85603": {
-    "commit": "e51caab73241b5c1598dfac5acbb2bde291a5e1f",
-    "text": "kubeadm: make sure images are pre-pulled even if a tag did not change but their contents changed",
-    "markdown": "kubeadm: make sure images are pre-pulled even if a tag did not change but their contents changed ([#85603](https://github.com/kubernetes/kubernetes/pull/85603), [@bart0sh](https://github.com/bart0sh))",
-    "author": "bart0sh",
-    "author_url": "https://github.com/bart0sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85603",
-    "pr_number": 85603,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "85609": {
-    "commit": "7907c63fad1ccb7905d2d27c643387046ab0f4ad",
-    "text": "kubeadm now supports automatic calculations of dual-stack node cidr masks to kube-controller-manager.",
-    "markdown": "kubeadm now supports automatic calculations of dual-stack node cidr masks to kube-controller-manager. ([#85609](https://github.com/kubernetes/kubernetes/pull/85609), [@Arvinderpal](https://github.com/Arvinderpal))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "Arvinderpal",
-    "author_url": "https://github.com/Arvinderpal",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85609",
-    "pr_number": 85609,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.17.0"
-  },
-  "85610": {
-    "commit": "82ee37f3e1ef2be3bfca8031e9babfc748084b4f",
-    "text": "Update Cluster Autoscaler to 1.17.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.17.0",
-    "markdown": "Update Cluster Autoscaler to 1.17.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.17.0 ([#85610](https://github.com/kubernetes/kubernetes/pull/85610), [@losipiuk](https://github.com/losipiuk))",
-    "author": "losipiuk",
-    "author_url": "https://github.com/losipiuk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85610",
-    "pr_number": 85610,
-    "kinds": ["bug"],
-    "sigs": ["autoscaling", "cluster-lifecycle"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85639": {
-    "commit": "674695c544296c90e0a4fa0abdab71b3679a82c3",
-    "text": "kubeadm now errors out whenever a not supported component config version is supplied for the kubelet and kube-proxy",
-    "markdown": "kubeadm now errors out whenever a not supported component config version is supplied for the kubelet and kube-proxy ([#85639](https://github.com/kubernetes/kubernetes/pull/85639), [@rosti](https://github.com/rosti))",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-cluster-lifecycle/kubeadm/20190925-component-configs.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85639",
-    "pr_number": 85639,
-    "areas": ["kubeadm"],
-    "kinds": ["design"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "85675": {
-    "commit": "b920b388e90d32ea268b1b23142b65514fe6139e",
-    "text": "Fixed \"requested device X but found Y\" attach error on AWS.",
-    "markdown": "Fixed \"requested device X but found Y\" attach error on AWS. ([#85675](https://github.com/kubernetes/kubernetes/pull/85675), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85675",
-    "pr_number": 85675,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85722": {
-    "commit": "1d343c8a0ce3e4d4845f7c34d06a1e2f25c975b7",
-    "text": "Filter published OpenAPI schema by making nullable, required fields non-required in order to avoid kubectl to wrongly reject null values.",
-    "markdown": "Filter published OpenAPI schema by making nullable, required fields non-required in order to avoid kubectl to wrongly reject null values. ([#85722](https://github.com/kubernetes/kubernetes/pull/85722), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85722",
-    "pr_number": 85722,
-    "areas": ["custom-resources"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.0"
-  },
-  "85745": {
-    "commit": "3769de9401474ea22c0ddbacde896911ddd87700",
-    "text": "kubeadm: uses the apiserver AdvertiseAddress IP family to choose the etcd endpoint IP family for non external etcd clusters",
-    "markdown": "kubeadm: uses the apiserver AdvertiseAddress IP family to choose the etcd endpoint IP family for non external etcd clusters ([#85745](https://github.com/kubernetes/kubernetes/pull/85745), [@aojea](https://github.com/aojea))",
-    "author": "aojea",
-    "author_url": "https://github.com/aojea",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85745",
-    "pr_number": 85745,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "85763": {
-    "commit": "1ca289dfc11547af7b1e7aa6c6fd09972260a7c6",
-    "text": "kubeadm: retry `kubeadm-config` ConfigMap creation or mutation if the apiserver is not responding. This will improve resiliency when joining new control plane nodes.",
-    "markdown": "kubeadm: retry `kubeadm-config` ConfigMap creation or mutation if the apiserver is not responding. This will improve resiliency when joining new control plane nodes. ([#85763](https://github.com/kubernetes/kubernetes/pull/85763), [@ereslibre](https://github.com/ereslibre))",
-    "author": "ereslibre",
-    "author_url": "https://github.com/ereslibre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85763",
-    "pr_number": 85763,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  },
-  "85793": {
-    "commit": "85b5a046f7de104281225485797232d6cdd71389",
-    "text": "Fixed issue with addon-resizer using deprecated extensions APIs",
-    "markdown": "Fixed issue with addon-resizer using deprecated extensions APIs ([#85793](https://github.com/kubernetes/kubernetes/pull/85793), [@bskiba](https://github.com/bskiba))",
-    "author": "bskiba",
-    "author_url": "https://github.com/bskiba",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85793",
-    "pr_number": 85793,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85810": {
-    "commit": "19ee1ea9fa61b2a3fe4d86438bef5203bfe1a756",
-    "text": "Resolved regression in admission, authentication, and authorization webhook performance in v1.17.0-rc.1",
-    "markdown": "Resolved regression in admission, authentication, and authorization webhook performance in v1.17.0-rc.1 ([#85810](https://github.com/kubernetes/kubernetes/pull/85810), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85810",
-    "pr_number": 85810,
-    "areas": ["apiserver", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.0"
-  },
-  "85817": {
-    "commit": "6e0ce9fb2ff8f26d1becafedfc732d603363b9a7",
-    "text": "kubeadm: Forward cluster name to the controller-manager arguments",
-    "markdown": "kubeadm: Forward cluster name to the controller-manager arguments ([#85817](https://github.com/kubernetes/kubernetes/pull/85817), [@ereslibre](https://github.com/ereslibre))",
-    "author": "ereslibre",
-    "author_url": "https://github.com/ereslibre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85817",
-    "pr_number": 85817,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.0"
-  }
-}
diff --git a/src/assets/release-notes-1.17.1.json b/src/assets/release-notes-1.17.1.json
deleted file mode 100644
index 194df47d..00000000
--- a/src/assets/release-notes-1.17.1.json
+++ /dev/null
@@ -1,1024 +0,0 @@
-{
-  "73363": {
-    "commit": "67647ed5a379ea72d7db7b5461e9802aa1debdbf",
-    "text": "Minimize AWS NLB health check timeout when externalTrafficPolicy set to Local",
-    "markdown": "Minimize AWS NLB health check timeout when externalTrafficPolicy set to Local ([#73363](https://github.com/kubernetes/kubernetes/pull/73363), [@kellycampbell](https://github.com/kellycampbell))",
-    "author": "kellycampbell",
-    "author_url": "https://github.com/kellycampbell",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73363",
-    "pr_number": 73363,
-    "areas": ["cloudprovider", "provider/aws"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.17.1"
-  },
-  "74525": {
-    "commit": "928817a26a84d9e3076d110ea30ba994912aa477",
-    "text": "Added the HPA API, that allows scale behavior to be configured through the HPA\n`behavior` field. Behaviors are specified separately for scaling up and down. In\neach direction a stabilization window can be specified as well as a list of\npolicies and how to select amongst them. Policies can limit the absolute number\nof pods added or removed, or the percentage of pods added or removed.",
-    "markdown": "Added the HPA API, that allows scale behavior to be configured through the HPA\n  `behavior` field. Behaviors are specified separately for scaling up and down. In\n  each direction a stabilization window can be specified as well as a list of\n  policies and how to select amongst them. Policies can limit the absolute number\n  of pods added or removed, or the percentage of pods added or removed. ([#74525](https://github.com/kubernetes/kubernetes/pull/74525), [@gliush](https://github.com/gliush))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Autoscaling, and SIG CLI",
-    "author": "gliush",
-    "author_url": "https://github.com/gliush",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74525",
-    "pr_number": 74525,
-    "areas": ["kubectl"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "autoscaling", "cli"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "80649": {
-    "commit": "bbd48e99b2c19d999dfa25320f055a20deabbc38",
-    "text": "API request throttling (due to a high rate of requests) is now reported in the kubelet (and other component) logs by default.  The messages are of the form\n\nThrottling request took 1.50705208s, request: GET:\u003cURL\u003e\n\nThe presence of large numbers of these messages, particularly with long delay times, may indicate to the administrator the need to tune the cluster accordingly.",
-    "markdown": "API request throttling (due to a high rate of requests) is now reported in the kubelet (and other component) logs by default.  The messages are of the form\n  \n  Throttling request took 1.50705208s, request: GET:\u003cURL\u003e\n  \n  The presence of large numbers of these messages, particularly with long delay times, may indicate to the administrator the need to tune the cluster accordingly. ([#80649](https://github.com/kubernetes/kubernetes/pull/80649), [@RobertKrawitz](https://github.com/RobertKrawitz))\n\n  Courtesy of SIG API Machinery, and SIG Node",
-    "author": "RobertKrawitz",
-    "author_url": "https://github.com/RobertKrawitz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80649",
-    "pr_number": 80649,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "node"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "82462": {
-    "commit": "5373fa3f593a12309ff0435858991dd117bdb6ca",
-    "text": "kube-proxy: Added dual-stack IPv4/IPv6 support to the iptables proxier.",
-    "markdown": "kube-proxy: Added dual-stack IPv4/IPv6 support to the iptables proxier. ([#82462](https://github.com/kubernetes/kubernetes/pull/82462), [@vllry](https://github.com/vllry))\n\n  Courtesy of SIG Network",
-    "author": "vllry",
-    "author_url": "https://github.com/vllry",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82462",
-    "pr_number": 82462,
-    "areas": ["ipvs"],
-    "kinds": ["feature"],
-    "sigs": ["network"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "83358": {
-    "commit": "0c026656fe1c697594a756e543578ff3eab98060",
-    "text": "Bind kube-dns containers to linux nodes to avoid Windows scheduling",
-    "markdown": "Bind kube-dns containers to linux nodes to avoid Windows scheduling ([#83358](https://github.com/kubernetes/kubernetes/pull/83358), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83358",
-    "pr_number": 83358,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.17.1"
-  },
-  "83362": {
-    "commit": "082883cba32c61b69bdb6709d0132217b4d73dc6",
-    "text": "Bind metrics-server containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes",
-    "markdown": "Bind metrics-server containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes ([#83362](https://github.com/kubernetes/kubernetes/pull/83362), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83362",
-    "pr_number": 83362,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "instrumentation", "windows"],
-    "duplicate": true,
-    "release_version": "1.17.1"
-  },
-  "83363": {
-    "commit": "332f452198a8d2fbb2ca20e937a050cf213fac01",
-    "text": "Bind metadata-agent containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes",
-    "markdown": "Bind metadata-agent containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes ([#83363](https://github.com/kubernetes/kubernetes/pull/83363), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83363",
-    "pr_number": 83363,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "instrumentation", "windows"],
-    "duplicate": true,
-    "release_version": "1.17.1"
-  },
-  "83364": {
-    "commit": "fc98aa54465dbb31e2260cc51fb753c8eae55a9e",
-    "text": "Bind dns-horizontal containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes",
-    "markdown": "Bind dns-horizontal containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes ([#83364](https://github.com/kubernetes/kubernetes/pull/83364), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83364",
-    "pr_number": 83364,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.17.1"
-  },
-  "83437": {
-    "commit": "9fa1e00be9f4816e18d463f8125a7f20c0d4aee1",
-    "text": "Promote StartupProbe to beta for 1.18 release",
-    "markdown": "Promote StartupProbe to beta for 1.18 release ([#83437](https://github.com/kubernetes/kubernetes/pull/83437), [@matthyx](https://github.com/matthyx))\n\n  Courtesy of SIG Node, SIG Scalability, and SIG Testing",
-    "author": "matthyx",
-    "author_url": "https://github.com/matthyx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83437",
-    "pr_number": 83437,
-    "areas": ["test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["node", "scalability", "testing"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "83841": {
-    "commit": "ff8cf507dc5476832408ab6a679ffacccfaf2696",
-    "text": "Following metrics have been turned off:\n- kubelet_pod_worker_latency_microseconds\n- kubelet_pod_start_latency_microseconds\n- kubelet_cgroup_manager_latency_microseconds\n- kubelet_pod_worker_start_latency_microseconds\n- kubelet_pleg_relist_latency_microseconds\n- kubelet_pleg_relist_interval_microseconds\n- kubelet_eviction_stats_age_microseconds\n- kubelet_runtime_operations\n- kubelet_runtime_operations_latency_microseconds\n- kubelet_runtime_operations_errors\n- kubelet_device_plugin_registration_count\n- kubelet_device_plugin_alloc_latency_microseconds\n- kubelet_docker_operations\n- kubelet_docker_operations_latency_microseconds\n- kubelet_docker_operations_errors\n- kubelet_docker_operations_timeout\n- network_plugin_operations_latency_microseconds",
-    "markdown": "Following metrics have been turned off:\n  - kubelet_pod_worker_latency_microseconds\n  - kubelet_pod_start_latency_microseconds\n  - kubelet_cgroup_manager_latency_microseconds\n  - kubelet_pod_worker_start_latency_microseconds\n  - kubelet_pleg_relist_latency_microseconds\n  - kubelet_pleg_relist_interval_microseconds\n  - kubelet_eviction_stats_age_microseconds\n  - kubelet_runtime_operations\n  - kubelet_runtime_operations_latency_microseconds\n  - kubelet_runtime_operations_errors\n  - kubelet_device_plugin_registration_count\n  - kubelet_device_plugin_alloc_latency_microseconds\n  - kubelet_docker_operations\n  - kubelet_docker_operations_latency_microseconds\n  - kubelet_docker_operations_errors\n  - kubelet_docker_operations_timeout\n  - network_plugin_operations_latency_microseconds ([#83841](https://github.com/kubernetes/kubernetes/pull/83841), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Network, and SIG Node",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/da4b7050ccae7a947e4d60f94ab28513e513a458/keps/sig-instrumentation/20181106-kubernetes-metrics-overhaul.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83841",
-    "pr_number": 83841,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["network", "node"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "84163": {
-    "commit": "daaa0b9c6ba2dbf37cc0ae4464ed8b35521c7b62",
-    "text": "Bug fixes:\nMake sure we include latest packages node \u0026#35;351 (@caseydavenport)",
-    "markdown": "Bug fixes:\n  Make sure we include latest packages node \u0026#35;351 (@caseydavenport) ([#84163](https://github.com/kubernetes/kubernetes/pull/84163), [@david-tigera](https://github.com/david-tigera))",
-    "author": "david-tigera",
-    "author_url": "https://github.com/david-tigera",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84163",
-    "pr_number": 84163,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.1"
-  },
-  "84462": {
-    "commit": "a1fc96f41e943645229379a6d459826123c11c94",
-    "text": "The underlying format of the `CPUManager` state file has changed. Upgrades should be seamless, but any third-party tools that rely on reading the previous format need to be updated.",
-    "markdown": "The underlying format of the `CPUManager` state file has changed. Upgrades should be seamless, but any third-party tools that rely on reading the previous format need to be updated. ([#84462](https://github.com/kubernetes/kubernetes/pull/84462), [@klueska](https://github.com/klueska))\n\n  Courtesy of SIG Node, and SIG Testing",
-    "author": "klueska",
-    "author_url": "https://github.com/klueska",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84462",
-    "pr_number": 84462,
-    "areas": ["kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["node", "testing"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "84735": {
-    "commit": "7fdefe50026895f74c7f4685ff2de90f712f05f7",
-    "text": "The sample-apiserver aggregated conformance test has updated to use the Kubernetes v1.17.0 sample apiserver",
-    "markdown": "The sample-apiserver aggregated conformance test has updated to use the Kubernetes v1.17.0 sample apiserver ([#84735](https://github.com/kubernetes/kubernetes/pull/84735), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84735",
-    "pr_number": 84735,
-    "areas": ["conformance", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "architecture", "cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.1"
-  },
-  "84886": {
-    "commit": "78c56e6b4cf963e5d773908ef9780345d216188b",
-    "text": "Verify kubelet \u0026 kube-proxy can recover after being killed on Windows nodes",
-    "markdown": "Verify kubelet \u0026 kube-proxy can recover after being killed on Windows nodes ([#84886](https://github.com/kubernetes/kubernetes/pull/84886), [@YangLu1031](https://github.com/YangLu1031))",
-    "author": "YangLu1031",
-    "author_url": "https://github.com/YangLu1031",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84886",
-    "pr_number": 84886,
-    "areas": ["test"],
-    "kinds": ["failing-test"],
-    "sigs": ["apps", "testing", "windows"],
-    "duplicate": true,
-    "release_version": "1.17.1"
-  },
-  "84951": {
-    "commit": "010291d4dc3f20d4c5f26f7e04704b9db0659b54",
-    "text": "Fixed a regression where the kubelet would fail to update the ready status of pods.",
-    "markdown": "Fixed a regression where the kubelet would fail to update the ready status of pods. ([#84951](https://github.com/kubernetes/kubernetes/pull/84951), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84951",
-    "pr_number": 84951,
-    "areas": ["kubelet"],
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.17.1"
-  },
-  "85279": {
-    "commit": "2796ff85388112d22ca0145d7762be7beb429e07",
-    "text": "New flag `--show-hidden-metrics-for-version` in kube-proxy can be used to show all hidden metrics that deprecated in the previous minor release.",
-    "markdown": "New flag `--show-hidden-metrics-for-version` in kube-proxy can be used to show all hidden metrics that deprecated in the previous minor release. ([#85279](https://github.com/kubernetes/kubernetes/pull/85279), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Network",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85279",
-    "pr_number": 85279,
-    "areas": ["kubeadm"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "85308": {
-    "commit": "a26fcf5278a7f643c3aa1dce9be60a3240bb1a72",
-    "text": "If a serving certificates param specifies a name that is an IP for an SNI certificate, it will have priority for replying to server connections.",
-    "markdown": "If a serving certificates param specifies a name that is an IP for an SNI certificate, it will have priority for replying to server connections. ([#85308](https://github.com/kubernetes/kubernetes/pull/85308), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85308",
-    "pr_number": 85308,
-    "areas": ["apiserver", "dependency"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.1"
-  },
-  "85410": {
-    "commit": "34f349257f4fc14c1adc8d9a62109216a6955035",
-    "text": "Fix API Server potential memory leak issue in processing watch request.",
-    "markdown": "Fix API Server potential memory leak issue in processing watch request. ([#85410](https://github.com/kubernetes/kubernetes/pull/85410), [@answer1991](https://github.com/answer1991))",
-    "author": "answer1991",
-    "author_url": "https://github.com/answer1991",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85410",
-    "pr_number": 85410,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.1"
-  },
-  "85446": {
-    "commit": "c7a65ca0c352d58d94327ed80f35757bef3d6792",
-    "text": "kubelet: the metric process_start_time_seconds be marked as with the ALPHA stability level.",
-    "markdown": "kubelet: the metric process_start_time_seconds be marked as with the ALPHA stability level. ([#85446](https://github.com/kubernetes/kubernetes/pull/85446), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG API Machinery, SIG Cluster Lifecycle, SIG Instrumentation, and SIG Node",
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85446",
-    "pr_number": 85446,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "instrumentation", "node"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "85463": {
-    "commit": "4000732bec17d16ad183e38774883f8a6ae93fd4",
-    "text": "SafeSysctlWhitelist: add net.ipv4.ping_group_range",
-    "markdown": "SafeSysctlWhitelist: add net.ipv4.ping_group_range ([#85463](https://github.com/kubernetes/kubernetes/pull/85463), [@AkihiroSuda](https://github.com/AkihiroSuda))\n\n  Courtesy of SIG Auth",
-    "author": "AkihiroSuda",
-    "author_url": "https://github.com/AkihiroSuda",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85463",
-    "pr_number": 85463,
-    "kinds": ["feature"],
-    "sigs": ["auth"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "85511": {
-    "commit": "92a14f4f7ef11c580961f0aebe491d0ff148e191",
-    "text": "Fix a bug in port-forward: named port not working with service",
-    "markdown": "Fix a bug in port-forward: named port not working with service ([#85511](https://github.com/kubernetes/kubernetes/pull/85511), [@oke-py](https://github.com/oke-py))",
-    "author": "oke-py",
-    "author_url": "https://github.com/oke-py",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85511",
-    "pr_number": 85511,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.17.1"
-  },
-  "85512": {
-    "commit": "127c47caf49bec4a59a0500f99b1b1d4cdbf340e",
-    "text": "Remove cluster-monitoring addon",
-    "markdown": "Remove cluster-monitoring addon ([#85512](https://github.com/kubernetes/kubernetes/pull/85512), [@serathius](https://github.com/serathius))",
-    "author": "serathius",
-    "author_url": "https://github.com/serathius",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85512",
-    "pr_number": 85512,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["cleanup", "deprecation"],
-    "sigs": ["cluster-lifecycle", "instrumentation", "scalability", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.1"
-  },
-  "85517": {
-    "commit": "e1c711227691bd2b32519e86513bbad88b65c972",
-    "text": "add kube-proxy flags --ipvs-tcp-timeout, --ipvs-tcpfin-timeout, --ipvs-udp-timeout to configure IPVS connection timeouts.",
-    "markdown": "add kube-proxy flags --ipvs-tcp-timeout, --ipvs-tcpfin-timeout, --ipvs-udp-timeout to configure IPVS connection timeouts. ([#85517](https://github.com/kubernetes/kubernetes/pull/85517), [@andrewsykim](https://github.com/andrewsykim))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Network",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85517",
-    "pr_number": 85517,
-    "areas": ["ipvs", "kubeadm"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "85542": {
-    "commit": "541b4e4b395b5124f3517b4e62f3dc604016db11",
-    "text": "e2e-test-framework: add e2e test namespace dump if all tests succeed but the cleanup fails.",
-    "markdown": "e2e-test-framework: add e2e test namespace dump if all tests succeed but the cleanup fails. ([#85542](https://github.com/kubernetes/kubernetes/pull/85542), [@schrodit](https://github.com/schrodit))",
-    "author": "schrodit",
-    "author_url": "https://github.com/schrodit",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85542",
-    "pr_number": 85542,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["bug", "failing-test"],
-    "sigs": ["testing"],
-    "release_version": "1.17.1"
-  },
-  "85577": {
-    "commit": "93eb2e38a7800087f0ab2ae058cccd4dcceae8f9",
-    "text": "kubectl/drain: add skip-wait-for-delete-timeout option.\nIf pod DeletionTimestamp older than N seconds, skip waiting for the pod.  Seconds must be greater than 0 to skip.",
-    "markdown": "kubectl/drain: add skip-wait-for-delete-timeout option.\n  If pod DeletionTimestamp older than N seconds, skip waiting for the pod.  Seconds must be greater than 0 to skip. ([#85577](https://github.com/kubernetes/kubernetes/pull/85577), [@michaelgugino](https://github.com/michaelgugino))\n\n  Courtesy of SIG CLI",
-    "documentation": [
-      {
-        "description": "[Other doc]",
-        "url": "https://github.com/kubernetes/enhancements/issues/1339",
-        "type": "KEP"
-      }
-    ],
-    "author": "michaelgugino",
-    "author_url": "https://github.com/michaelgugino",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85577",
-    "pr_number": 85577,
-    "areas": ["kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "85703": {
-    "commit": "d4ba10ec808aec7c3529b4b216d428e024861225",
-    "text": "Fix EndpointSlice controller race condition and ensure that it handles external changes to EndpointSlices.",
-    "markdown": "Fix EndpointSlice controller race condition and ensure that it handles external changes to EndpointSlices. ([#85703](https://github.com/kubernetes/kubernetes/pull/85703), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85703",
-    "pr_number": 85703,
-    "kinds": ["bug"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.17.1"
-  },
-  "85739": {
-    "commit": "220e2ceba0784fe230d590d58e26d87cbf05af13",
-    "text": "Update go-winio module version from 0.4.11 to 0.4.14",
-    "markdown": "Update go-winio module version from 0.4.11 to 0.4.14 ([#85739](https://github.com/kubernetes/kubernetes/pull/85739), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85739",
-    "pr_number": 85739,
-    "areas": ["dependency"],
-    "kinds": ["bug"],
-    "sigs": ["windows"],
-    "release_version": "1.17.1"
-  },
-  "85747": {
-    "commit": "0ab15832400c762c7203721d551d2072e2951c6d",
-    "text": "Fixes wrong validation result of NetworkPolicy PolicyTypes",
-    "markdown": "Fixes wrong validation result of NetworkPolicy PolicyTypes ([#85747](https://github.com/kubernetes/kubernetes/pull/85747), [@tnqn](https://github.com/tnqn))",
-    "author": "tnqn",
-    "author_url": "https://github.com/tnqn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85747",
-    "pr_number": 85747,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.17.1"
-  },
-  "85771": {
-    "commit": "d0e901807d4c11e725b7b3ae95d312a13d5dd7f8",
-    "text": "kubernetes will try to acquire the iptables lock every 100 msec during 5 seconds instead of every second. This specially useful for environments using kube-proxy in iptables mode with a high churn rate of services.",
-    "markdown": "kubernetes will try to acquire the iptables lock every 100 msec during 5 seconds instead of every second. This specially useful for environments using kube-proxy in iptables mode with a high churn rate of services. ([#85771](https://github.com/kubernetes/kubernetes/pull/85771), [@aojea](https://github.com/aojea))",
-    "author": "aojea",
-    "author_url": "https://github.com/aojea",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85771",
-    "pr_number": 85771,
-    "kinds": ["flake"],
-    "sigs": ["network"],
-    "release_version": "1.17.1"
-  },
-  "85789": {
-    "commit": "6db550d1db70e389463930bccfb4855f7369b815",
-    "text": "OwnerReferences []OwnerReferencematches \nso nil len(OwnerReferences) ==0 \n\nchange:\n if lease.OwnerReferences == nil || len(lease.OwnerReferences) == 0 ==\u003eif len(lease.OwnerReferences) == 0",
-    "markdown": "OwnerReferences []OwnerReferencematches \n  so nil len(OwnerReferences) ==0 \n  \n  change:\n   if lease.OwnerReferences == nil || len(lease.OwnerReferences) == 0 ==\u003eif len(lease.OwnerReferences) == 0 ([#85789](https://github.com/kubernetes/kubernetes/pull/85789), [@ZP-AlwaysWin](https://github.com/ZP-AlwaysWin))",
-    "author": "ZP-AlwaysWin",
-    "author_url": "https://github.com/ZP-AlwaysWin",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85789",
-    "pr_number": 85789,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.17.1"
-  },
-  "85836": {
-    "commit": "8727eef79c679bf40ff0cfa3318dadbc5fd50e24",
-    "text": "Add instructions about how to bring up e2e test cluster",
-    "markdown": "Add instructions about how to bring up e2e test cluster ([#85836](https://github.com/kubernetes/kubernetes/pull/85836), [@YangLu1031](https://github.com/YangLu1031))",
-    "author": "YangLu1031",
-    "author_url": "https://github.com/YangLu1031",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85836",
-    "pr_number": 85836,
-    "kinds": ["documentation"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.1"
-  },
-  "85837": {
-    "commit": "7b792c38e0c9b66413d790236c80367df3094484",
-    "text": "allow for configuration of CoreDNS replica count",
-    "markdown": "allow for configuration of CoreDNS replica count ([#85837](https://github.com/kubernetes/kubernetes/pull/85837), [@pickledrick](https://github.com/pickledrick))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "pickledrick",
-    "author_url": "https://github.com/pickledrick",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85837",
-    "pr_number": 85837,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "85857": {
-    "commit": "2fbe432d2347d7f808054f92f2b146e8a7dd2de8",
-    "text": "During initialization phase (preflight), kubeadm now verifies the presence of the conntrack executable",
-    "markdown": "During initialization phase (preflight), kubeadm now verifies the presence of the conntrack executable ([#85857](https://github.com/kubernetes/kubernetes/pull/85857), [@hnanni](https://github.com/hnanni))",
-    "author": "hnanni",
-    "author_url": "https://github.com/hnanni",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85857",
-    "pr_number": 85857,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.1"
-  },
-  "85874": {
-    "commit": "398e2bcc734f04686ff89fdd9f5b47cf12cd525b",
-    "text": "- Renamed Kubelet metric certificate_manager_server_expiration_seconds to certificate_manager_server_ttl_seconds and changed to report the second until expiration at read time rather than absolute time of expiry.\n- Improved accuracy of Kubelet metric rest_client_exec_plugin_ttl_seconds.",
-    "markdown": "- Renamed Kubelet metric certificate_manager_server_expiration_seconds to certificate_manager_server_ttl_seconds and changed to report the second until expiration at read time rather than absolute time of expiry.\n  - Improved accuracy of Kubelet metric rest_client_exec_plugin_ttl_seconds. ([#85874](https://github.com/kubernetes/kubernetes/pull/85874), [@sambdavidson](https://github.com/sambdavidson))",
-    "author": "sambdavidson",
-    "author_url": "https://github.com/sambdavidson",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85874",
-    "pr_number": 85874,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "auth", "cluster-lifecycle", "instrumentation", "node"],
-    "duplicate": true,
-    "release_version": "1.17.1"
-  },
-  "85879": {
-    "commit": "814fc34cde8c51ffc83870e18ad703deece63af7",
-    "text": "hollow-node use fake CRI",
-    "markdown": "hollow-node use fake CRI ([#85879](https://github.com/kubernetes/kubernetes/pull/85879), [@gongguan](https://github.com/gongguan))\n\n  Courtesy of SIG Node, and SIG Scalability",
-    "author": "gongguan",
-    "author_url": "https://github.com/gongguan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85879",
-    "pr_number": 85879,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node", "scalability"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "85885": {
-    "commit": "50437b4c5da29baba1f8aeef201c2b7736cd42d2",
-    "text": "VMSS cache is added so that less chances of VMSS GET throttling",
-    "markdown": "VMSS cache is added so that less chances of VMSS GET throttling ([#85885](https://github.com/kubernetes/kubernetes/pull/85885), [@nilo19](https://github.com/nilo19))",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85885",
-    "pr_number": 85885,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.17.1"
-  },
-  "85903": {
-    "commit": "6a919f2b54bc3fbc047250f0aa8c423f5209c2b5",
-    "text": "kube-apiserver no longer serves the following deprecated APIs:\n- All resources under `apps/v1beta1` and `apps/v1beta2` - use `apps/v1` instead\n- `daemonsets`, `deployments`, `replicasets` resources under `extensions/v1beta1` - use `apps/v1` instead\n- `networkpolicies` resources under `extensions/v1beta1` - use `networking.k8s.io/v1` instead\n- `podsecuritypolicies` resources under `extensions/v1beta1` - use `policy/v1beta1` instead",
-    "markdown": "kube-apiserver no longer serves the following deprecated APIs:\n  - All resources under `apps/v1beta1` and `apps/v1beta2` - use `apps/v1` instead\n  - `daemonsets`, `deployments`, `replicasets` resources under `extensions/v1beta1` - use `apps/v1` instead\n  - `networkpolicies` resources under `extensions/v1beta1` - use `networking.k8s.io/v1` instead\n  - `podsecuritypolicies` resources under `extensions/v1beta1` - use `policy/v1beta1` instead ([#85903](https://github.com/kubernetes/kubernetes/pull/85903), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85903",
-    "pr_number": 85903,
-    "areas": ["apiserver", "test"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["api-machinery", "apps", "cluster-lifecycle", "instrumentation", "testing"],
-    "duplicate": true,
-    "release_version": "1.17.1"
-  },
-  "85944": {
-    "commit": "a233e0115bb56e014f812365f583704075accc7b",
-    "text": "elasticsearch supports automatically setting the advertise address",
-    "markdown": "elasticsearch supports automatically setting the advertise address ([#85944](https://github.com/kubernetes/kubernetes/pull/85944), [@SataQiu](https://github.com/SataQiu))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Instrumentation",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85944",
-    "pr_number": 85944,
-    "kinds": ["bug", "feature"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "85975": {
-    "commit": "6f61072606155b070b2f79850b008685df3a74ff",
-    "text": "Fix nil pointer dereference in azure cloud provider",
-    "markdown": "Fix nil pointer dereference in azure cloud provider ([#85975](https://github.com/kubernetes/kubernetes/pull/85975), [@ldx](https://github.com/ldx))",
-    "author": "ldx",
-    "author_url": "https://github.com/ldx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85975",
-    "pr_number": 85975,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.17.1"
-  },
-  "85990": {
-    "commit": "b5040be796b71f07de2f0be782d14e9b24b018b7",
-    "text": "Fix LoadBalancer rule checking so that no unexpected LoadBalancer updates are made",
-    "markdown": "Fix LoadBalancer rule checking so that no unexpected LoadBalancer updates are made ([#85990](https://github.com/kubernetes/kubernetes/pull/85990), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85990",
-    "pr_number": 85990,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.17.1"
-  },
-  "85993": {
-    "commit": "7a50fdb2a635f4f3219f6411fa42d89c65a77800",
-    "text": "fix a regression in kubenet that prevent pods to obtain ip addresses",
-    "markdown": "fix a regression in kubenet that prevent pods to obtain ip addresses ([#85993](https://github.com/kubernetes/kubernetes/pull/85993), [@chendotjs](https://github.com/chendotjs))",
-    "author": "chendotjs",
-    "author_url": "https://github.com/chendotjs",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85993",
-    "pr_number": 85993,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["network", "node"],
-    "duplicate": true,
-    "release_version": "1.17.1"
-  },
-  "86001": {
-    "commit": "ed8a260fb3ff03843e3e29bc8e0f24fe4bfb7ab4",
-    "text": "Fix IPv6 addresses lost issue in pure ipv6 vsphere environment",
-    "markdown": "Fix IPv6 addresses lost issue in pure ipv6 vsphere environment ([#86001](https://github.com/kubernetes/kubernetes/pull/86001), [@hubv](https://github.com/hubv))",
-    "author": "hubv",
-    "author_url": "https://github.com/hubv",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86001",
-    "pr_number": 86001,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.17.1"
-  },
-  "86016": {
-    "commit": "078344c12524adfe1f6f742e0d5a21016d65e64b",
-    "text": "Fixes kube-proxy when EndpointSlice feature gate is enabled on Windows.",
-    "markdown": "Fixes kube-proxy when EndpointSlice feature gate is enabled on Windows. ([#86016](https://github.com/kubernetes/kubernetes/pull/86016), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86016",
-    "pr_number": 86016,
-    "kinds": ["bug"],
-    "sigs": ["auth", "network"],
-    "duplicate": true,
-    "release_version": "1.17.1"
-  },
-  "86022": {
-    "commit": "d842c194c21f86ddcecae303ce9d8dcce882a18c",
-    "text": "Fixed an issue that the scheduler only returns the first failure reason.",
-    "markdown": "Fixed an issue that the scheduler only returns the first failure reason. ([#86022](https://github.com/kubernetes/kubernetes/pull/86022), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86022",
-    "pr_number": 86022,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.17.1"
-  },
-  "86078": {
-    "commit": "939da5e9cf7a041fd5cdf992449372e7b9833162",
-    "text": "Make error message and service event message more clear",
-    "markdown": "Make error message and service event message more clear ([#86078](https://github.com/kubernetes/kubernetes/pull/86078), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86078",
-    "pr_number": 86078,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.17.1"
-  },
-  "86092": {
-    "commit": "442107b6b96a980e4d26c30a176071092748414e",
-    "text": "kube-proxy no longer modifies shared EndpointSlices.",
-    "markdown": "kube-proxy no longer modifies shared EndpointSlices. ([#86092](https://github.com/kubernetes/kubernetes/pull/86092), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86092",
-    "pr_number": 86092,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.17.1"
-  },
-  "86167": {
-    "commit": "03e90b80ceebc4197eae7cc6b95a59623bd9a51f",
-    "text": "change CounterVec to Counter about  PLEGDiscardEvent",
-    "markdown": "change CounterVec to Counter about  PLEGDiscardEvent ([#86167](https://github.com/kubernetes/kubernetes/pull/86167), [@yiyang5055](https://github.com/yiyang5055))",
-    "author": "yiyang5055",
-    "author_url": "https://github.com/yiyang5055",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86167",
-    "pr_number": 86167,
-    "kinds": ["cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.17.1"
-  },
-  "86168": {
-    "commit": "0d58709016f3358f34a52c12087d707a64cc6cb0",
-    "text": "Resolves performance regression in `kubectl get all` and in client-go discovery clients constructed using `NewDiscoveryClientForConfig` or `NewDiscoveryClientForConfigOrDie`.",
-    "markdown": "Resolves performance regression in `kubectl get all` and in client-go discovery clients constructed using `NewDiscoveryClientForConfig` or `NewDiscoveryClientForConfigOrDie`. ([#86168](https://github.com/kubernetes/kubernetes/pull/86168), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86168",
-    "pr_number": 86168,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.1"
-  },
-  "86210": {
-    "commit": "3ccdad175a80a73c8a756e2f35b7ac1d6a98d106",
-    "text": "The following features are unconditionally enabled and the corresponding `--feature-gates` flags have been removed: `PodPriority`, `TaintNodesByCondition`, `ResourceQuotaScopeSelectors` and `ScheduleDaemonSetPods`",
-    "markdown": "The following features are unconditionally enabled and the corresponding `--feature-gates` flags have been removed: `PodPriority`, `TaintNodesByCondition`, `ResourceQuotaScopeSelectors` and `ScheduleDaemonSetPods` ([#86210](https://github.com/kubernetes/kubernetes/pull/86210), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86210",
-    "pr_number": 86210,
-    "kinds": ["cleanup"],
-    "sigs": ["apps", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.17.1"
-  },
-  "86216": {
-    "commit": "e622579b143730b89d471ecd4e5f6761e9c37d86",
-    "text": "kubeadm: fix potential panic when executing \"kubeadm reset\" with a corrupted kubelet.conf file",
-    "markdown": "kubeadm: fix potential panic when executing \"kubeadm reset\" with a corrupted kubelet.conf file ([#86216](https://github.com/kubernetes/kubernetes/pull/86216), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86216",
-    "pr_number": 86216,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.1"
-  },
-  "86251": {
-    "commit": "49bc6966149a92e02aaee0245881c6a4a83dc007",
-    "text": "New metric kubelet_pleg_last_seen_seconds to aid diagnosis of PLEG not healthy issues.",
-    "markdown": "New metric kubelet_pleg_last_seen_seconds to aid diagnosis of PLEG not healthy issues. ([#86251](https://github.com/kubernetes/kubernetes/pull/86251), [@bboreham](https://github.com/bboreham))\n\n  Courtesy of SIG Node",
-    "author": "bboreham",
-    "author_url": "https://github.com/bboreham",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86251",
-    "pr_number": 86251,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "86266": {
-    "commit": "f06a3800ed7819fa68aa30054b4cd52619a4cbde",
-    "text": "azure cloud provider cache TTL is configurable, list of the azure cloud provider is as following:\n- \"availabilitySetNodesCacheTTLInSeconds\"\n- \"vmssCacheTTLInSeconds\"\n- \"vmssVirtualMachinesCacheTTLInSeconds\"\n- \"vmCacheTTLInSeconds\"\n- \"loadBalancerCacheTTLInSeconds\"\n- \"nsgCacheTTLInSeconds\"\n- \"routeTableCacheTTLInSeconds\"",
-    "markdown": "azure cloud provider cache TTL is configurable, list of the azure cloud provider is as following:\n  - \"availabilitySetNodesCacheTTLInSeconds\"\n  - \"vmssCacheTTLInSeconds\"\n  - \"vmssVirtualMachinesCacheTTLInSeconds\"\n  - \"vmCacheTTLInSeconds\"\n  - \"loadBalancerCacheTTLInSeconds\"\n  - \"nsgCacheTTLInSeconds\"\n  - \"routeTableCacheTTLInSeconds\" ([#86266](https://github.com/kubernetes/kubernetes/pull/86266), [@zqingqing1](https://github.com/zqingqing1))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "zqingqing1",
-    "author_url": "https://github.com/zqingqing1",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86266",
-    "pr_number": 86266,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "86277": {
-    "commit": "a57a8f33b323cd002e76c664bf0f91f71df7c927",
-    "text": "Fixed a panic in the kubelet cleaning up pod volumes",
-    "markdown": "Fixed a panic in the kubelet cleaning up pod volumes ([#86277](https://github.com/kubernetes/kubernetes/pull/86277), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86277",
-    "pr_number": 86277,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.17.1"
-  },
-  "86320": {
-    "commit": "eee9dee7b9c0ad926c8619b211902f118feff2fa",
-    "text": "Fixes an issue with kubelet-reported pod status on deleted/recreated pods.",
-    "markdown": "Fixes an issue with kubelet-reported pod status on deleted/recreated pods. ([#86320](https://github.com/kubernetes/kubernetes/pull/86320), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86320",
-    "pr_number": 86320,
-    "areas": ["kubelet"],
-    "kinds": ["bug", "flake"],
-    "sigs": ["node"],
-    "release_version": "1.17.1"
-  },
-  "86329": {
-    "commit": "4a62b3ac6dc7937ea798f2163cb3bb1f1a165f20",
-    "text": "Changed core_pattern on COS nodes to be an absolute path.",
-    "markdown": "Changed core_pattern on COS nodes to be an absolute path. ([#86329](https://github.com/kubernetes/kubernetes/pull/86329), [@mml](https://github.com/mml))",
-    "author": "mml",
-    "author_url": "https://github.com/mml",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86329",
-    "pr_number": 86329,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "node"],
-    "duplicate": true,
-    "release_version": "1.17.1"
-  },
-  "86338": {
-    "commit": "6f4ec3e0d6e1c1e9947091aedcaee882753363d6",
-    "text": "option `preConfiguredBackendPoolLoadBalancerTypes` is added to azure cloud provider for the pre-configured load balancers, possible values: `\"\"`, `\"internal\"`, \"external\"`, `\"all\"`",
-    "markdown": "option `preConfiguredBackendPoolLoadBalancerTypes` is added to azure cloud provider for the pre-configured load balancers, possible values: `\"\"`, `\"internal\"`, \"external\"`, `\"all\"` ([#86338](https://github.com/kubernetes/kubernetes/pull/86338), [@gossion](https://github.com/gossion))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "gossion",
-    "author_url": "https://github.com/gossion",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86338",
-    "pr_number": 86338,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "86351": {
-    "commit": "ac4079a5cc22ced66c868f6fd26f8e2497765e51",
-    "text": "fix: azure data disk should use same key as os disk by default",
-    "markdown": "fix: azure data disk should use same key as os disk by default ([#86351](https://github.com/kubernetes/kubernetes/pull/86351), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86351",
-    "pr_number": 86351,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.17.1"
-  },
-  "86369": {
-    "commit": "84198bf1206c92a14a5f237c4161d5138886492e",
-    "text": "`AlwaysCheckAllPredicates` is deprecated in scheduler Policy API.",
-    "markdown": "`AlwaysCheckAllPredicates` is deprecated in scheduler Policy API. ([#86369](https://github.com/kubernetes/kubernetes/pull/86369), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86369",
-    "pr_number": 86369,
-    "kinds": ["api-change", "cleanup", "deprecation"],
-    "sigs": ["scheduling"],
-    "release_version": "1.17.1"
-  },
-  "86412": {
-    "commit": "3577447764103cc3204b66307d651ac2be0ec83f",
-    "text": "Fixes issue where AAD token obtained by kubectl is incompatible with on-behalf-of flow and oidc.\nThe audience claim before this fix has \"spn:\" prefix. After this fix, \"spn:\" prefix is omitted.",
-    "markdown": "Fixes issue where AAD token obtained by kubectl is incompatible with on-behalf-of flow and oidc.\n  The audience claim before this fix has \"spn:\" prefix. After this fix, \"spn:\" prefix is omitted. ([#86412](https://github.com/kubernetes/kubernetes/pull/86412), [@weinong](https://github.com/weinong))",
-    "author": "weinong",
-    "author_url": "https://github.com/weinong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86412",
-    "pr_number": 86412,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.17.1"
-  },
-  "86425": {
-    "commit": "9f841bf42396daf1ed4b7ab47c1e1708bfb028b3",
-    "text": "hollow-node do not use remote CRI anymore",
-    "markdown": "hollow-node do not use remote CRI anymore ([#86425](https://github.com/kubernetes/kubernetes/pull/86425), [@jkaniuk](https://github.com/jkaniuk))",
-    "author": "jkaniuk",
-    "author_url": "https://github.com/jkaniuk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86425",
-    "pr_number": 86425,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "scalability"],
-    "duplicate": true,
-    "release_version": "1.17.1"
-  },
-  "86493": {
-    "commit": "3760d24674d9eac92284c5d19c4a774aceab1b7f",
-    "text": "kubeadm: use bind-address option to configure the kube-controller-manager and kube-scheduler http probes",
-    "markdown": "kubeadm: use bind-address option to configure the kube-controller-manager and kube-scheduler http probes ([#86493](https://github.com/kubernetes/kubernetes/pull/86493), [@aojea](https://github.com/aojea))",
-    "author": "aojea",
-    "author_url": "https://github.com/aojea",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86493",
-    "pr_number": 86493,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.1"
-  },
-  "86496": {
-    "commit": "06fb3eb582187931a22c9fc57024063d91d965f8",
-    "text": "Adds back support for AlwaysCheckAllPredicates flag.",
-    "markdown": "Adds back support for AlwaysCheckAllPredicates flag. ([#86496](https://github.com/kubernetes/kubernetes/pull/86496), [@ahg-g](https://github.com/ahg-g))",
-    "author": "ahg-g",
-    "author_url": "https://github.com/ahg-g",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86496",
-    "pr_number": 86496,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.17.1"
-  },
-  "86502": {
-    "commit": "4f70231ce7736cc748f76526c98955f86c667a41",
-    "text": "Fix issue \u0026#35;85805 about resource not found in azure cloud provider when lb specified in other resource group.",
-    "markdown": "Fix issue \u0026#35;85805 about resource not found in azure cloud provider when lb specified in other resource group. ([#86502](https://github.com/kubernetes/kubernetes/pull/86502), [@levimm](https://github.com/levimm))",
-    "author": "levimm",
-    "author_url": "https://github.com/levimm",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86502",
-    "pr_number": 86502,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.17.1"
-  },
-  "86515": {
-    "commit": "92e0de32698864f4dce876d79e80386261e3e18c",
-    "text": "Azure global rate limit is switched to per-client. A set of new rate limit configure options are introduced, including routeRateLimit, SubnetsRateLimit, InterfaceRateLimit, RouteTableRateLimit, LoadBalancerRateLimit, PublicIPAddressRateLimit, SecurityGroupRateLimit, VirtualMachineRateLimit, StorageAccountRateLimit, DiskRateLimit, SnapshotRateLimit, VirtualMachineScaleSetRateLimit and VirtualMachineSizeRateLimit.\n\nThe original rate limit options would be default values for those new client's rate limiter.",
-    "markdown": "Azure global rate limit is switched to per-client. A set of new rate limit configure options are introduced, including routeRateLimit, SubnetsRateLimit, InterfaceRateLimit, RouteTableRateLimit, LoadBalancerRateLimit, PublicIPAddressRateLimit, SecurityGroupRateLimit, VirtualMachineRateLimit, StorageAccountRateLimit, DiskRateLimit, SnapshotRateLimit, VirtualMachineScaleSetRateLimit and VirtualMachineSizeRateLimit.\n  \n  The original rate limit options would be default values for those new client's rate limiter. ([#86515](https://github.com/kubernetes/kubernetes/pull/86515), [@feiskyer](https://github.com/feiskyer))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86515",
-    "pr_number": 86515,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "86534": {
-    "commit": "5e31799701123c50025567b8534e1a62dbc0e9f6",
-    "text": "Fixes v1.17.0 regression in --service-cluster-ip-range handling with IPv4 ranges larger than 65536 IP addresses",
-    "markdown": "Fixes v1.17.0 regression in --service-cluster-ip-range handling with IPv4 ranges larger than 65536 IP addresses ([#86534](https://github.com/kubernetes/kubernetes/pull/86534), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86534",
-    "pr_number": 86534,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.17.1"
-  },
-  "86584": {
-    "commit": "180e1c9f5159a22576ee866a82a7dc8c26cc1c41",
-    "text": "Marked scheduler's metrics scheduling_algorithm_predicate_evaluation_seconds and\nscheduling_algorithm_priority_evaluation_seconds as deprecated. Those are replaced by framework_extension_point_duration_seconds[extenstion_point=\"Filter\"] and framework_extension_point_duration_seconds[extenstion_point=\"Score\"] respectively.",
-    "markdown": "Marked scheduler's metrics scheduling_algorithm_predicate_evaluation_seconds and\n  scheduling_algorithm_priority_evaluation_seconds as deprecated. Those are replaced by framework_extension_point_duration_seconds[extenstion_point=\"Filter\"] and framework_extension_point_duration_seconds[extenstion_point=\"Score\"] respectively. ([#86584](https://github.com/kubernetes/kubernetes/pull/86584), [@xiaoanyunfei](https://github.com/xiaoanyunfei))",
-    "author": "xiaoanyunfei",
-    "author_url": "https://github.com/xiaoanyunfei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86584",
-    "pr_number": 86584,
-    "kinds": ["cleanup", "deprecation"],
-    "sigs": ["scheduling"],
-    "release_version": "1.17.1"
-  },
-  "86586": {
-    "commit": "fdfcb00edee127df1f9d27f0b858271ed8959421",
-    "text": "Marked scheduler's scheduling_duration_seconds Summary metric as deprecated",
-    "markdown": "Marked scheduler's scheduling_duration_seconds Summary metric as deprecated ([#86586](https://github.com/kubernetes/kubernetes/pull/86586), [@xiaoanyunfei](https://github.com/xiaoanyunfei))",
-    "author": "xiaoanyunfei",
-    "author_url": "https://github.com/xiaoanyunfei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86586",
-    "pr_number": 86586,
-    "kinds": ["cleanup", "deprecation"],
-    "sigs": ["scheduling"],
-    "release_version": "1.17.1"
-  },
-  "86609": {
-    "commit": "f772c9017c16bbc983e59a19162814629a467cef",
-    "text": "kubeadm: add support for the \"ci/k8s-master\" version label as a replacement for \"ci-cross/*\", which no longer exists.",
-    "markdown": "kubeadm: add support for the \"ci/k8s-master\" version label as a replacement for \"ci-cross/*\", which no longer exists. ([#86609](https://github.com/kubernetes/kubernetes/pull/86609), [@Pensu](https://github.com/Pensu))",
-    "author": "Pensu",
-    "author_url": "https://github.com/Pensu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86609",
-    "pr_number": 86609,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.1"
-  },
-  "86612": {
-    "commit": "40320be679e9bc85632a621dc2fa12af39f37c6d",
-    "text": "fix: azure disk could not mounted on Standard_DC4s/DC2s instances",
-    "markdown": "fix: azure disk could not mounted on Standard_DC4s/DC2s instances ([#86612](https://github.com/kubernetes/kubernetes/pull/86612), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86612",
-    "pr_number": 86612,
-    "areas": ["provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.17.1"
-  },
-  "86646": {
-    "commit": "19ecd690fa1ff0747db6388714c0569dfc2548bd",
-    "text": "For subprotocol negotiation, both client and server protocol is required now.",
-    "markdown": "For subprotocol negotiation, both client and server protocol is required now. ([#86646](https://github.com/kubernetes/kubernetes/pull/86646), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86646",
-    "pr_number": 86646,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "node"],
-    "duplicate": true,
-    "release_version": "1.17.1"
-  },
-  "86693": {
-    "commit": "c72d5fd2556f90669fc3321061dba658cad077cf",
-    "text": "If a required flag is not provided to a command, the user will only see the required flag error message, instead of the entire usage menu.",
-    "markdown": "If a required flag is not provided to a command, the user will only see the required flag error message, instead of the entire usage menu. ([#86693](https://github.com/kubernetes/kubernetes/pull/86693), [@sallyom](https://github.com/sallyom))",
-    "author": "sallyom",
-    "author_url": "https://github.com/sallyom",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86693",
-    "pr_number": 86693,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.17.1"
-  },
-  "86705": {
-    "commit": "09cb73a554d3218f0267ba28de8a5fba47fd6335",
-    "text": "kubeadm: tolerate whitespace when validating certificate authority PEM data in kubeconfig files",
-    "markdown": "kubeadm: tolerate whitespace when validating certificate authority PEM data in kubeconfig files ([#86705](https://github.com/kubernetes/kubernetes/pull/86705), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86705",
-    "pr_number": 86705,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.1"
-  },
-  "86740": {
-    "commit": "b046dbce1332f6d67e8c3adf03a7b62f605b759c",
-    "text": "Azure VMSS/VMSSVM clients now suppress requests on throttling",
-    "markdown": "Azure VMSS/VMSSVM clients now suppress requests on throttling ([#86740](https://github.com/kubernetes/kubernetes/pull/86740), [@feiskyer](https://github.com/feiskyer))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86740",
-    "pr_number": 86740,
-    "areas": ["cloudprovider", "dependency", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "86940": {
-    "commit": "edad4bbfc824215fc254096dfbbd1b2ab8ce6781",
-    "text": "Add VolumeBinder method to FrameworkHandle interface, which allows user to get the volume binder when implementing scheduler framework plugins.",
-    "markdown": "Add VolumeBinder method to FrameworkHandle interface, which allows user to get the volume binder when implementing scheduler framework plugins. ([#86940](https://github.com/kubernetes/kubernetes/pull/86940), [@skilxn-go](https://github.com/skilxn-go))\n\n  Courtesy of SIG Scheduling",
-    "author": "skilxn-go",
-    "author_url": "https://github.com/skilxn-go",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86940",
-    "pr_number": 86940,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.17.1"
-  },
-  "86952": {
-    "commit": "6026a6a1515244855d10a51857af2e8c2fd26f37",
-    "text": "fix kubectl annotate error when local=true is set",
-    "markdown": "fix kubectl annotate error when local=true is set ([#86952](https://github.com/kubernetes/kubernetes/pull/86952), [@zhouya0](https://github.com/zhouya0))",
-    "author": "zhouya0",
-    "author_url": "https://github.com/zhouya0",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86952",
-    "pr_number": 86952,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.17.1"
-  },
-  "87043": {
-    "commit": "240782c7f7ee1a810207b532f970ad0fe5812939",
-    "text": "Fixed a bug which could prevent a provider ID from ever being set for node if an error occurred determining the provider ID when the node was added.",
-    "markdown": "Fixed a bug which could prevent a provider ID from ever being set for node if an error occurred determining the provider ID when the node was added. ([#87043](https://github.com/kubernetes/kubernetes/pull/87043), [@zjs](https://github.com/zjs))",
-    "author": "zjs",
-    "author_url": "https://github.com/zjs",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87043",
-    "pr_number": 87043,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.17.1"
-  }
-}
diff --git a/src/assets/release-notes-1.17.2.json b/src/assets/release-notes-1.17.2.json
deleted file mode 100644
index f625d008..00000000
--- a/src/assets/release-notes-1.17.2.json
+++ /dev/null
@@ -1,253 +0,0 @@
-{
-  "76496": {
-    "commit": "f4db8212be53c69a27d893d6a4111422fbce8008",
-    "text": "The following deprecated metrics are removed, please convert to the corresponding metrics:\n1. The following replacement metrics are available from v1.14.0:\n- `rest_client_request_latency_seconds` -\u003e `rest_client_request_duration_seconds`\n- `scheduler_scheduling_latency_seconds` -\u003e `scheduler_scheduling_duration_seconds `\n- `docker_operations` -\u003e `docker_operations_total`\n- `docker_operations_latency_microseconds` -\u003e `docker_operations_duration_seconds`\n- `docker_operations_errors` -\u003e `docker_operations_errors_total`\n- `docker_operations_timeout` -\u003e `docker_operations_timeout_total`\n- `network_plugin_operations_latency_microseconds` -\u003e `network_plugin_operations_duration_seconds`\n- `kubelet_pod_worker_latency_microseconds` -\u003e `kubelet_pod_worker_duration_seconds`\n- `kubelet_pod_start_latency_microseconds` -\u003e `kubelet_pod_start_duration_seconds`\n- `kubelet_cgroup_manager_latency_microseconds` -\u003e `kubelet_cgroup_manager_duration_seconds`\n- `kubelet_pod_worker_start_latency_microseconds` -\u003e `kubelet_pod_worker_start_duration_seconds`\n- `kubelet_pleg_relist_latency_microseconds` -\u003e `kubelet_pleg_relist_duration_seconds`\n- `kubelet_pleg_relist_interval_microseconds` -\u003e `kubelet_pleg_relist_interval_seconds`\n- `kubelet_eviction_stats_age_microseconds` -\u003e `kubelet_eviction_stats_age_seconds`\n- `kubelet_runtime_operations` -\u003e `kubelet_runtime_operations_total`\n- `kubelet_runtime_operations_latency_microseconds` -\u003e `kubelet_runtime_operations_duration_seconds`\n- `kubelet_runtime_operations_errors` -\u003e `kubelet_runtime_operations_errors_total`\n- `kubelet_device_plugin_registration_count` -\u003e `kubelet_device_plugin_registration_total`\n- `kubelet_device_plugin_alloc_latency_microseconds` -\u003e `kubelet_device_plugin_alloc_duration_seconds`\n- `scheduler_e2e_scheduling_latency_microseconds` -\u003e `scheduler_e2e_scheduling_duration_seconds`\n- `scheduler_scheduling_algorithm_latency_microseconds` -\u003e `scheduler_scheduling_algorithm_duration_seconds`\n- `scheduler_scheduling_algorithm_predicate_evaluation` -\u003e `scheduler_scheduling_algorithm_predicate_evaluation_seconds`\n- `scheduler_scheduling_algorithm_priority_evaluation` -\u003e `scheduler_scheduling_algorithm_priority_evaluation_seconds`\n- `scheduler_scheduling_algorithm_preemption_evaluation` -\u003e `scheduler_scheduling_algorithm_preemption_evaluation_seconds`\n- `scheduler_binding_latency_microseconds` -\u003e `scheduler_binding_duration_seconds`\n- `kubeproxy_sync_proxy_rules_latency_microseconds` -\u003e `kubeproxy_sync_proxy_rules_duration_seconds`\n- `apiserver_request_latencies` -\u003e `apiserver_request_duration_seconds`\n- `apiserver_dropped_requests` -\u003e `apiserver_dropped_requests_total`\n- `etcd_request_latencies_summary` -\u003e `etcd_request_duration_seconds`\n- `apiserver_storage_transformation_latencies_microseconds ` -\u003e `apiserver_storage_transformation_duration_seconds`\n- `apiserver_storage_data_key_generation_latencies_microseconds` -\u003e `apiserver_storage_data_key_generation_duration_seconds`\n- `apiserver_request_count` -\u003e `apiserver_request_total`\n- `apiserver_request_latencies_summary`\n2. The following replacement metrics are available from v1.15.0:\n- `apiserver_storage_transformation_failures_total` -\u003e `apiserver_storage_transformation_operations_total`",
-    "markdown": "The following deprecated metrics are removed, please convert to the corresponding metrics:\n  1. The following replacement metrics are available from v1.14.0:\n  - `rest_client_request_latency_seconds` -\u003e `rest_client_request_duration_seconds`\n  - `scheduler_scheduling_latency_seconds` -\u003e `scheduler_scheduling_duration_seconds `\n  - `docker_operations` -\u003e `docker_operations_total`\n  - `docker_operations_latency_microseconds` -\u003e `docker_operations_duration_seconds`\n  - `docker_operations_errors` -\u003e `docker_operations_errors_total`\n  - `docker_operations_timeout` -\u003e `docker_operations_timeout_total`\n  - `network_plugin_operations_latency_microseconds` -\u003e `network_plugin_operations_duration_seconds`\n  - `kubelet_pod_worker_latency_microseconds` -\u003e `kubelet_pod_worker_duration_seconds`\n  - `kubelet_pod_start_latency_microseconds` -\u003e `kubelet_pod_start_duration_seconds`\n  - `kubelet_cgroup_manager_latency_microseconds` -\u003e `kubelet_cgroup_manager_duration_seconds`\n  - `kubelet_pod_worker_start_latency_microseconds` -\u003e `kubelet_pod_worker_start_duration_seconds`\n  - `kubelet_pleg_relist_latency_microseconds` -\u003e `kubelet_pleg_relist_duration_seconds`\n  - `kubelet_pleg_relist_interval_microseconds` -\u003e `kubelet_pleg_relist_interval_seconds`\n  - `kubelet_eviction_stats_age_microseconds` -\u003e `kubelet_eviction_stats_age_seconds`\n  - `kubelet_runtime_operations` -\u003e `kubelet_runtime_operations_total`\n  - `kubelet_runtime_operations_latency_microseconds` -\u003e `kubelet_runtime_operations_duration_seconds`\n  - `kubelet_runtime_operations_errors` -\u003e `kubelet_runtime_operations_errors_total`\n  - `kubelet_device_plugin_registration_count` -\u003e `kubelet_device_plugin_registration_total`\n  - `kubelet_device_plugin_alloc_latency_microseconds` -\u003e `kubelet_device_plugin_alloc_duration_seconds`\n  - `scheduler_e2e_scheduling_latency_microseconds` -\u003e `scheduler_e2e_scheduling_duration_seconds`\n  - `scheduler_scheduling_algorithm_latency_microseconds` -\u003e `scheduler_scheduling_algorithm_duration_seconds`\n  - `scheduler_scheduling_algorithm_predicate_evaluation` -\u003e `scheduler_scheduling_algorithm_predicate_evaluation_seconds`\n  - `scheduler_scheduling_algorithm_priority_evaluation` -\u003e `scheduler_scheduling_algorithm_priority_evaluation_seconds`\n  - `scheduler_scheduling_algorithm_preemption_evaluation` -\u003e `scheduler_scheduling_algorithm_preemption_evaluation_seconds`\n  - `scheduler_binding_latency_microseconds` -\u003e `scheduler_binding_duration_seconds`\n  - `kubeproxy_sync_proxy_rules_latency_microseconds` -\u003e `kubeproxy_sync_proxy_rules_duration_seconds`\n  - `apiserver_request_latencies` -\u003e `apiserver_request_duration_seconds`\n  - `apiserver_dropped_requests` -\u003e `apiserver_dropped_requests_total`\n  - `etcd_request_latencies_summary` -\u003e `etcd_request_duration_seconds`\n  - `apiserver_storage_transformation_latencies_microseconds ` -\u003e `apiserver_storage_transformation_duration_seconds`\n  - `apiserver_storage_data_key_generation_latencies_microseconds` -\u003e `apiserver_storage_data_key_generation_duration_seconds`\n  - `apiserver_request_count` -\u003e `apiserver_request_total`\n  - `apiserver_request_latencies_summary`\n  2. The following replacement metrics are available from v1.15.0:\n  - `apiserver_storage_transformation_failures_total` -\u003e `apiserver_storage_transformation_operations_total` ([#76496](https://github.com/kubernetes/kubernetes/pull/76496), [@danielqsj](https://github.com/danielqsj))",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76496",
-    "pr_number": 76496,
-    "areas": ["apiserver", "ipvs", "kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": [
-      "api-machinery",
-      "cluster-lifecycle",
-      "instrumentation",
-      "network",
-      "node",
-      "scheduling"
-    ],
-    "duplicate": true,
-    "release_version": "1.17.2"
-  },
-  "84265": {
-    "commit": "c9003a268dff6700506929b247847341ea4d5b33",
-    "text": "Fixed the following \n-  AWS Cloud Provider attempts to delete LoadBalancer security group it didn’t provision\n-  AWS Cloud Provider creates default LoadBalancer security group even if annotation [service.beta.kubernetes.io/aws-load-balancer-security-groups] is present",
-    "markdown": "Fixed the following \n  -  AWS Cloud Provider attempts to delete LoadBalancer security group it didn’t provision\n  -  AWS Cloud Provider creates default LoadBalancer security group even if annotation [service.beta.kubernetes.io/aws-load-balancer-security-groups] is present ([#84265](https://github.com/kubernetes/kubernetes/pull/84265), [@bhagwat070919](https://github.com/bhagwat070919))",
-    "author": "bhagwat070919",
-    "author_url": "https://github.com/bhagwat070919",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84265",
-    "pr_number": 84265,
-    "areas": ["cloudprovider", "provider/aws"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.17.2"
-  },
-  "85281": {
-    "commit": "d9bf22f8f7df184cf427114e8ad1fef12474364f",
-    "text": "New flag `--show-hidden-metrics-for-version` in kube-controller-manager can be used to show all hidden metrics that deprecated in the previous minor release.",
-    "markdown": "New flag `--show-hidden-metrics-for-version` in kube-controller-manager can be used to show all hidden metrics that deprecated in the previous minor release. ([#85281](https://github.com/kubernetes/kubernetes/pull/85281), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG API Machinery",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85281",
-    "pr_number": 85281,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.17.2"
-  },
-  "85283": {
-    "commit": "dd77283426bd3249004472efcbb3257744448161",
-    "text": "Cleaned up the output from `kubectl describe CSINode \u003cname\u003e`.",
-    "markdown": "Cleaned up the output from `kubectl describe CSINode \u003cname\u003e`. ([#85283](https://github.com/kubernetes/kubernetes/pull/85283), [@huffmanca](https://github.com/huffmanca))",
-    "author": "huffmanca",
-    "author_url": "https://github.com/huffmanca",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85283",
-    "pr_number": 85283,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli", "storage"],
-    "duplicate": true,
-    "release_version": "1.17.2"
-  },
-  "85458": {
-    "commit": "7f2128757b1aa2e83320d1f635fe8b2e7028542a",
-    "text": "Improved yaml parsing performance",
-    "markdown": "Improved yaml parsing performance ([#85458](https://github.com/kubernetes/kubernetes/pull/85458), [@cjcullen](https://github.com/cjcullen))",
-    "author": "cjcullen",
-    "author_url": "https://github.com/cjcullen",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85458",
-    "pr_number": 85458,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl"],
-    "kinds": ["bug"],
-    "sigs": [
-      "api-machinery",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "node"
-    ],
-    "duplicate": true,
-    "release_version": "1.17.2"
-  },
-  "85613": {
-    "commit": "05209312b74eac15f0a80577cbbdcb11bce9b8ad",
-    "text": "`kubectl apply  -f \u003cfile\u003e --prune -n \u003cnamespace\u003e` should prune all resources not defined in the file in the cli specified namespace.",
-    "markdown": "`kubectl apply  -f \u003cfile\u003e --prune -n \u003cnamespace\u003e` should prune all resources not defined in the file in the cli specified namespace. ([#85613](https://github.com/kubernetes/kubernetes/pull/85613), [@MartinKaburu](https://github.com/MartinKaburu))",
-    "author": "MartinKaburu",
-    "author_url": "https://github.com/MartinKaburu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85613",
-    "pr_number": 85613,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.17.2"
-  },
-  "85850": {
-    "commit": "61d36e4a43b831f960b190b81d371cb33b5f20d1",
-    "text": "You can now pass \"--node-ip ::\" to kubelet to indicate that it should autodetect an IPv6 address to use as the node's primary address.",
-    "markdown": "You can now pass \"--node-ip ::\" to kubelet to indicate that it should autodetect an IPv6 address to use as the node's primary address. ([#85850](https://github.com/kubernetes/kubernetes/pull/85850), [@danwinship](https://github.com/danwinship))\n\n  Courtesy of SIG Cloud Provider, SIG Network, and SIG Node",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85850",
-    "pr_number": 85850,
-    "areas": ["ipv6", "kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider", "network", "node"],
-    "feature": true,
-    "release_version": "1.17.2"
-  },
-  "86282": {
-    "commit": "be26fbc638ff5ed952ccae70e68baa712b3c60c8",
-    "text": "kubelet: resource metrics endpoint `/metrics/resource/v1alpha1` as well as all metrics under this endpoint have been deprecated.\nPlease convert to the following metrics emitted by endpoint `/metrics/resource`:\n- scrape_error --\u003e scrape_error\n- node_cpu_usage_seconds_total --\u003e node_cpu_usage_seconds\n- node_memory_working_set_bytes --\u003e node_memory_working_set_bytes\n- container_cpu_usage_seconds_total --\u003e container_cpu_usage_seconds\n- container_memory_working_set_bytes --\u003e container_memory_working_set_bytes\n- scrape_error --\u003e scrape_error",
-    "markdown": "kubelet: resource metrics endpoint `/metrics/resource/v1alpha1` as well as all metrics under this endpoint have been deprecated.\n  Please convert to the following metrics emitted by endpoint `/metrics/resource`:\n  - scrape_error --\u003e scrape_error\n  - node_cpu_usage_seconds_total --\u003e node_cpu_usage_seconds\n  - node_memory_working_set_bytes --\u003e node_memory_working_set_bytes\n  - container_cpu_usage_seconds_total --\u003e container_cpu_usage_seconds\n  - container_memory_working_set_bytes --\u003e container_memory_working_set_bytes\n  - scrape_error --\u003e scrape_error ([#86282](https://github.com/kubernetes/kubernetes/pull/86282), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Node",
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86282",
-    "pr_number": 86282,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.17.2"
-  },
-  "86861": {
-    "commit": "916edd922e528f2f27542d3ca5967ca11fe153f8",
-    "text": "kubeadm upgrades always persist the etcd backup for stacked",
-    "markdown": "kubeadm upgrades always persist the etcd backup for stacked ([#86861](https://github.com/kubernetes/kubernetes/pull/86861), [@SataQiu](https://github.com/SataQiu))",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86861",
-    "pr_number": 86861,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.17.2"
-  },
-  "86899": {
-    "commit": "3e8155ead3d890697141d4946ec5fdd5b0dab192",
-    "text": "kubeadm: support automatic retry after failing to pull image",
-    "markdown": "kubeadm: support automatic retry after failing to pull image ([#86899](https://github.com/kubernetes/kubernetes/pull/86899), [@SataQiu](https://github.com/SataQiu))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86899",
-    "pr_number": 86899,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.17.2"
-  },
-  "87029": {
-    "commit": "af46f2cdd2a72ef808f99715c978d58a4603871b",
-    "text": "Fixes service account token admission error in clusters that do not run the service account token controller",
-    "markdown": "Fixes service account token admission error in clusters that do not run the service account token controller ([#87029](https://github.com/kubernetes/kubernetes/pull/87029), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87029",
-    "pr_number": 87029,
-    "kinds": ["bug"],
-    "sigs": ["auth"],
-    "release_version": "1.17.2"
-  },
-  "87035": {
-    "commit": "d2a526bc0eb0961966584749935e7a28bd769adc",
-    "text": "provider/azure: Network security groups can now be in a separate resource group.",
-    "markdown": "provider/azure: Network security groups can now be in a separate resource group. ([#87035](https://github.com/kubernetes/kubernetes/pull/87035), [@CecileRobertMichon](https://github.com/CecileRobertMichon))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "CecileRobertMichon",
-    "author_url": "https://github.com/CecileRobertMichon",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87035",
-    "pr_number": 87035,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.17.2"
-  },
-  "87044": {
-    "commit": "36e40fb850293076b415ae3d376f5f81dc897105",
-    "text": "TODO",
-    "markdown": "TODO ([#87044](https://github.com/kubernetes/kubernetes/pull/87044), [@jennybuckley](https://github.com/jennybuckley))\n\n  Courtesy of SIG API Machinery",
-    "author": "jennybuckley",
-    "author_url": "https://github.com/jennybuckley",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87044",
-    "pr_number": 87044,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.17.2"
-  },
-  "87122": {
-    "commit": "93595eb4eb933bfed90b1f4a1f5597caccf7ccfd",
-    "text": "Azure network and VM clients now suppress requests on throttling",
-    "markdown": "Azure network and VM clients now suppress requests on throttling ([#87122](https://github.com/kubernetes/kubernetes/pull/87122), [@feiskyer](https://github.com/feiskyer))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87122",
-    "pr_number": 87122,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.17.2"
-  },
-  "87203": {
-    "commit": "0c8dcc59f099f69240ffd83eb2eae1cb6e1c68aa",
-    "text": "All apiservers log request lines in a more greppable format.",
-    "markdown": "All apiservers log request lines in a more greppable format. ([#87203](https://github.com/kubernetes/kubernetes/pull/87203), [@lavalamp](https://github.com/lavalamp))",
-    "author": "lavalamp",
-    "author_url": "https://github.com/lavalamp",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87203",
-    "pr_number": 87203,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.17.2"
-  },
-  "87213": {
-    "commit": "a25c6aa35067187800ef2a0928646f5d23268764",
-    "text": "CustomResourceDefinition status fields are no longer required for client validation when submitting manifests.",
-    "markdown": "CustomResourceDefinition status fields are no longer required for client validation when submitting manifests. ([#87213](https://github.com/kubernetes/kubernetes/pull/87213), [@hasheddan](https://github.com/hasheddan))\n\n  Courtesy of SIG API Machinery",
-    "author": "hasheddan",
-    "author_url": "https://github.com/hasheddan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87213",
-    "pr_number": 87213,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.17.2"
-  },
-  "87246": {
-    "commit": "6413f1ee2be99fe5ba18844a2646fb15d462bec4",
-    "text": "Fix the bug PIP's DNS is deleted if no DNS label service annotation isn't set.",
-    "markdown": "Fix the bug PIP's DNS is deleted if no DNS label service annotation isn't set. ([#87246](https://github.com/kubernetes/kubernetes/pull/87246), [@nilo19](https://github.com/nilo19))",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87246",
-    "pr_number": 87246,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.17.2"
-  }
-}
diff --git a/src/assets/release-notes-1.18.0-alpha.1.json b/src/assets/release-notes-1.18.0-alpha.1.json
deleted file mode 100644
index 98e9d874..00000000
--- a/src/assets/release-notes-1.18.0-alpha.1.json
+++ /dev/null
@@ -1,949 +0,0 @@
-{
-  "74525": {
-    "commit": "928817a26a84d9e3076d110ea30ba994912aa477",
-    "text": "TBA",
-    "markdown": "TBA ([#74525](https://github.com/kubernetes/kubernetes/pull/74525), [@gliush](https://github.com/gliush))\n\n  Courtesy of SIG API Machinery, SIG Apps, SIG Autoscaling, and SIG CLI",
-    "author": "gliush",
-    "author_url": "https://github.com/gliush",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/74525",
-    "pr_number": 74525,
-    "areas": ["kubectl"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery", "apps", "autoscaling", "cli"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "80649": {
-    "commit": "bbd48e99b2c19d999dfa25320f055a20deabbc38",
-    "text": "API request throttling (due to a high rate of requests) is now reported in the kubelet (and other component) logs by default.  The messages are of the form\n\nThrottling request took 1.50705208s, request: GET:\u003cURL\u003e\n\nThe presence of large numbers of these messages, particularly with long delay times, may indicate to the administrator the need to tune the cluster accordingly.",
-    "markdown": "API request throttling (due to a high rate of requests) is now reported in the kubelet (and other component) logs by default.  The messages are of the form\n  \n  Throttling request took 1.50705208s, request: GET:\u003cURL\u003e\n  \n  The presence of large numbers of these messages, particularly with long delay times, may indicate to the administrator the need to tune the cluster accordingly. ([#80649](https://github.com/kubernetes/kubernetes/pull/80649), [@RobertKrawitz](https://github.com/RobertKrawitz))\n\n  Courtesy of SIG API Machinery, and SIG Node",
-    "author": "RobertKrawitz",
-    "author_url": "https://github.com/RobertKrawitz",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/80649",
-    "pr_number": 80649,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "node"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "81319": {
-    "commit": "2bc3804a0e492aa21a368bde054f150edda2a346",
-    "text": "kubeadm: add a upgrade health check that deploys a Job",
-    "markdown": "kubeadm: add a upgrade health check that deploys a Job ([#81319](https://github.com/kubernetes/kubernetes/pull/81319), [@neolit123](https://github.com/neolit123))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/81319",
-    "pr_number": 81319,
-    "areas": ["kubeadm", "security"],
-    "kinds": ["cleanup", "feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "82660": {
-    "commit": "646afd52ff93b069401a61dc3b4b9ec37a1d936a",
-    "text": "kubectl drain node --dry-run will list pods that would be evicted or deleted",
-    "markdown": "kubectl drain node --dry-run will list pods that would be evicted or deleted ([#82660](https://github.com/kubernetes/kubernetes/pull/82660), [@sallyom](https://github.com/sallyom))",
-    "author": "sallyom",
-    "author_url": "https://github.com/sallyom",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82660",
-    "pr_number": 82660,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "83362": {
-    "commit": "082883cba32c61b69bdb6709d0132217b4d73dc6",
-    "text": "Bind metrics-server containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes",
-    "markdown": "Bind metrics-server containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes ([#83362](https://github.com/kubernetes/kubernetes/pull/83362), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83362",
-    "pr_number": 83362,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "instrumentation", "windows"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "83363": {
-    "commit": "332f452198a8d2fbb2ca20e937a050cf213fac01",
-    "text": "Bind metadata-agent containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes",
-    "markdown": "Bind metadata-agent containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes ([#83363](https://github.com/kubernetes/kubernetes/pull/83363), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83363",
-    "pr_number": 83363,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "instrumentation", "windows"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "83841": {
-    "commit": "ff8cf507dc5476832408ab6a679ffacccfaf2696",
-    "text": "Following metrics have been turned off:\n- kubelet_pod_worker_latency_microseconds\n- kubelet_pod_start_latency_microseconds\n- kubelet_cgroup_manager_latency_microseconds\n- kubelet_pod_worker_start_latency_microseconds\n- kubelet_pleg_relist_latency_microseconds\n- kubelet_pleg_relist_interval_microseconds\n- kubelet_eviction_stats_age_microseconds\n- kubelet_runtime_operations\n- kubelet_runtime_operations_latency_microseconds\n- kubelet_runtime_operations_errors\n- kubelet_device_plugin_registration_count\n- kubelet_device_plugin_alloc_latency_microseconds\n- kubelet_docker_operations\n- kubelet_docker_operations_latency_microseconds\n- kubelet_docker_operations_errors\n- kubelet_docker_operations_timeout\n- network_plugin_operations_latency_microseconds",
-    "markdown": "Following metrics have been turned off:\n  - kubelet_pod_worker_latency_microseconds\n  - kubelet_pod_start_latency_microseconds\n  - kubelet_cgroup_manager_latency_microseconds\n  - kubelet_pod_worker_start_latency_microseconds\n  - kubelet_pleg_relist_latency_microseconds\n  - kubelet_pleg_relist_interval_microseconds\n  - kubelet_eviction_stats_age_microseconds\n  - kubelet_runtime_operations\n  - kubelet_runtime_operations_latency_microseconds\n  - kubelet_runtime_operations_errors\n  - kubelet_device_plugin_registration_count\n  - kubelet_device_plugin_alloc_latency_microseconds\n  - kubelet_docker_operations\n  - kubelet_docker_operations_latency_microseconds\n  - kubelet_docker_operations_errors\n  - kubelet_docker_operations_timeout\n  - network_plugin_operations_latency_microseconds ([#83841](https://github.com/kubernetes/kubernetes/pull/83841), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Network, and SIG Node",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/da4b7050ccae7a947e4d60f94ab28513e513a458/keps/sig-instrumentation/20181106-kubernetes-metrics-overhaul.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83841",
-    "pr_number": 83841,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["network", "node"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "84462": {
-    "commit": "a1fc96f41e943645229379a6d459826123c11c94",
-    "text": "The underlying format of the `CPUManager` state file has changed. Upgrades should be seamless, but any third-party tools that rely on reading the previous format need to be updated.",
-    "markdown": "The underlying format of the `CPUManager` state file has changed. Upgrades should be seamless, but any third-party tools that rely on reading the previous format need to be updated. ([#84462](https://github.com/kubernetes/kubernetes/pull/84462), [@klueska](https://github.com/klueska))\n\n  Courtesy of SIG Node, and SIG Testing",
-    "author": "klueska",
-    "author_url": "https://github.com/klueska",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84462",
-    "pr_number": 84462,
-    "areas": ["kubelet", "test"],
-    "kinds": ["feature"],
-    "sigs": ["node", "testing"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "84614": {
-    "commit": "205570eb75b52bc0888309c71757314bd099b3d7",
-    "text": "kubelet now exports a \"server_expiration_renew_failure\" and \"client_expiration_renew_failure\" metric counter if the certificate rotations cannot be performed.",
-    "markdown": "kubelet now exports a \"server_expiration_renew_failure\" and \"client_expiration_renew_failure\" metric counter if the certificate rotations cannot be performed. ([#84614](https://github.com/kubernetes/kubernetes/pull/84614), [@rphillips](https://github.com/rphillips))\n\n  Courtesy of SIG API Machinery, SIG Auth, SIG CLI, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Instrumentation, SIG Node, and SIG Release",
-    "author": "rphillips",
-    "author_url": "https://github.com/rphillips",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84614",
-    "pr_number": 84614,
-    "areas": [
-      "apiserver",
-      "cloudprovider",
-      "code-generation",
-      "dependency",
-      "kubectl",
-      "kubelet",
-      "release-eng"
-    ],
-    "kinds": ["feature"],
-    "sigs": [
-      "api-machinery",
-      "auth",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "node",
-      "release"
-    ],
-    "feature": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "84735": {
-    "commit": "7fdefe50026895f74c7f4685ff2de90f712f05f7",
-    "text": "The sample-apiserver aggregated conformance test has updated to use the Kubernetes v1.17.0 sample apiserver",
-    "markdown": "The sample-apiserver aggregated conformance test has updated to use the Kubernetes v1.17.0 sample apiserver ([#84735](https://github.com/kubernetes/kubernetes/pull/84735), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84735",
-    "pr_number": 84735,
-    "areas": ["conformance", "test"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "architecture", "cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "84886": {
-    "commit": "78c56e6b4cf963e5d773908ef9780345d216188b",
-    "text": "Verify kubelet \u0026 kube-proxy can recover after being killed on Windows nodes",
-    "markdown": "Verify kubelet \u0026 kube-proxy can recover after being killed on Windows nodes ([#84886](https://github.com/kubernetes/kubernetes/pull/84886), [@YangLu1031](https://github.com/YangLu1031))",
-    "author": "YangLu1031",
-    "author_url": "https://github.com/YangLu1031",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84886",
-    "pr_number": 84886,
-    "areas": ["test"],
-    "kinds": ["failing-test"],
-    "sigs": ["apps", "testing", "windows"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "84951": {
-    "commit": "010291d4dc3f20d4c5f26f7e04704b9db0659b54",
-    "text": "Fixed a regression where the kubelet would fail to update the ready status of pods.",
-    "markdown": "Fixed a regression where the kubelet would fail to update the ready status of pods. ([#84951](https://github.com/kubernetes/kubernetes/pull/84951), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84951",
-    "pr_number": 84951,
-    "areas": ["kubelet"],
-    "kinds": ["bug", "cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85228": {
-    "commit": "02c61b6af1250bfabf99628870cfa689bd790c62",
-    "text": "Wait for kubelet \u0026 kube-proxy to be ready on Windows node within 10s",
-    "markdown": "Wait for kubelet \u0026 kube-proxy to be ready on Windows node within 10s ([#85228](https://github.com/kubernetes/kubernetes/pull/85228), [@YangLu1031](https://github.com/YangLu1031))",
-    "author": "YangLu1031",
-    "author_url": "https://github.com/YangLu1031",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85228",
-    "pr_number": 85228,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85265": {
-    "commit": "f01a4d06085b7b44844c472d6dedbd0538080c78",
-    "text": "kubeadm: reset raises warnings if it cannot delete folders",
-    "markdown": "kubeadm: reset raises warnings if it cannot delete folders ([#85265](https://github.com/kubernetes/kubernetes/pull/85265), [@SataQiu](https://github.com/SataQiu))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85265",
-    "pr_number": 85265,
-    "areas": ["kubeadm"],
-    "kinds": ["bug", "feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85279": {
-    "commit": "2796ff85388112d22ca0145d7762be7beb429e07",
-    "text": "New flag `--show-hidden-metrics-for-version` in kube-proxy can be used to show all hidden metrics that deprecated in the previous minor release.",
-    "markdown": "New flag `--show-hidden-metrics-for-version` in kube-proxy can be used to show all hidden metrics that deprecated in the previous minor release. ([#85279](https://github.com/kubernetes/kubernetes/pull/85279), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Network",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85279",
-    "pr_number": 85279,
-    "areas": ["kubeadm"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85293": {
-    "commit": "2b6857db4d4226cc07180308fe8a8507abbc7700",
-    "text": "Includes FSType when describing CSI persistent volumes.",
-    "markdown": "Includes FSType when describing CSI persistent volumes. ([#85293](https://github.com/kubernetes/kubernetes/pull/85293), [@huffmanca](https://github.com/huffmanca))",
-    "author": "huffmanca",
-    "author_url": "https://github.com/huffmanca",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85293",
-    "pr_number": 85293,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli", "storage"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85363": {
-    "commit": "0810bc33869b9b2e5bbabef4fa41e99d53526cdb",
-    "text": "action required\n1. Currently, if users were to explicitly specify CacheSize of 0 for KMS provider, they would end-up with a provider that caches up to 1000 keys. This PR changes this behavior.\nPost this PR, when users supply 0 for CacheSize this will result in a validation error.\n\n2. CacheSize type was changed from int32 to *int32. This allows defaulting logic to differentiate between cases where users explicitly supplied 0 vs. not supplied any value.\n3. KMS Provider's endpoint (path to Unix socket) is now validated when the EncryptionConfiguration files is loaded. This used to be handled by the GRPCService.",
-    "markdown": "action required\n  1. Currently, if users were to explicitly specify CacheSize of 0 for KMS provider, they would end-up with a provider that caches up to 1000 keys. This PR changes this behavior.\n  Post this PR, when users supply 0 for CacheSize this will result in a validation error.\n  \n  2. CacheSize type was changed from int32 to *int32. This allows defaulting logic to differentiate between cases where users explicitly supplied 0 vs. not supplied any value.\n  3. KMS Provider's endpoint (path to Unix socket) is now validated when the EncryptionConfiguration files is loaded. This used to be handled by the GRPCService. ([#85363](https://github.com/kubernetes/kubernetes/pull/85363), [@immutableT](https://github.com/immutableT))\n\n  Courtesy of SIG API Machinery, SIG Auth, and SIG Instrumentation",
-    "author": "immutableT",
-    "author_url": "https://github.com/immutableT",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85363",
-    "pr_number": 85363,
-    "areas": ["apiserver"],
-    "kinds": ["api-change"],
-    "sigs": ["api-machinery", "auth", "instrumentation"],
-    "action_required": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85368": {
-    "commit": "9905a33fd738a52d3f7ae73cf823d82a05720e33",
-    "text": "Fix bug where EndpointSlice controller would attempt to modify shared objects.",
-    "markdown": "Fix bug where EndpointSlice controller would attempt to modify shared objects. ([#85368](https://github.com/kubernetes/kubernetes/pull/85368), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85368",
-    "pr_number": 85368,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "apps", "network"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85408": {
-    "commit": "bf8c276019331f589dbdd2007e552a98d2e50eed",
-    "text": "Use context to check client closed instead of http.CloseNotifier in processing watch request which will reduce 1 goroutine for each request if proto is HTTP/2.x .",
-    "markdown": "Use context to check client closed instead of http.CloseNotifier in processing watch request which will reduce 1 goroutine for each request if proto is HTTP/2.x . ([#85408](https://github.com/kubernetes/kubernetes/pull/85408), [@answer1991](https://github.com/answer1991))",
-    "author": "answer1991",
-    "author_url": "https://github.com/answer1991",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85408",
-    "pr_number": 85408,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85410": {
-    "commit": "34f349257f4fc14c1adc8d9a62109216a6955035",
-    "text": "Fix API Server potential memory leak issue in processing watch request.",
-    "markdown": "Fix API Server potential memory leak issue in processing watch request. ([#85410](https://github.com/kubernetes/kubernetes/pull/85410), [@answer1991](https://github.com/answer1991))",
-    "author": "answer1991",
-    "author_url": "https://github.com/answer1991",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85410",
-    "pr_number": 85410,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85412": {
-    "commit": "c65c56348760b5a3c10d9ec214f99ddb3add1488",
-    "text": "kubeadm: don't write the kubelet environment file on \"upgrade apply\"",
-    "markdown": "kubeadm: don't write the kubelet environment file on \"upgrade apply\" ([#85412](https://github.com/kubernetes/kubernetes/pull/85412), [@boluisa](https://github.com/boluisa))",
-    "author": "boluisa",
-    "author_url": "https://github.com/boluisa",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85412",
-    "pr_number": 85412,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85444": {
-    "commit": "5d3125e51258646fff19089f9eb2f2f80c784385",
-    "text": "kube-apiserver: Fixes a bug that hidden metrics can not be enabled by the command-line option `--show-hidden-metrics-for-version`.",
-    "markdown": "kube-apiserver: Fixes a bug that hidden metrics can not be enabled by the command-line option `--show-hidden-metrics-for-version`. ([#85444](https://github.com/kubernetes/kubernetes/pull/85444), [@RainbowMango](https://github.com/RainbowMango))",
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85444",
-    "pr_number": 85444,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85446": {
-    "commit": "c7a65ca0c352d58d94327ed80f35757bef3d6792",
-    "text": "kubelet: the metric process_start_time_seconds be marked as with the ALPHA stability level.",
-    "markdown": "kubelet: the metric process_start_time_seconds be marked as with the ALPHA stability level. ([#85446](https://github.com/kubernetes/kubernetes/pull/85446), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG API Machinery, SIG Cluster Lifecycle, SIG Instrumentation, and SIG Node",
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85446",
-    "pr_number": 85446,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery", "cluster-lifecycle", "instrumentation", "node"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85463": {
-    "commit": "4000732bec17d16ad183e38774883f8a6ae93fd4",
-    "text": "SafeSysctlWhitelist: add net.ipv4.ping_group_range",
-    "markdown": "SafeSysctlWhitelist: add net.ipv4.ping_group_range ([#85463](https://github.com/kubernetes/kubernetes/pull/85463), [@AkihiroSuda](https://github.com/AkihiroSuda))\n\n  Courtesy of SIG Auth",
-    "author": "AkihiroSuda",
-    "author_url": "https://github.com/AkihiroSuda",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85463",
-    "pr_number": 85463,
-    "kinds": ["feature"],
-    "sigs": ["auth"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85466": {
-    "commit": "d9ed6110e88e73da597cc03a38ee1357f9b67ed3",
-    "text": "Windows nodes on GCE can use TPM-based authentication to the master.",
-    "markdown": "Windows nodes on GCE can use TPM-based authentication to the master. ([#85466](https://github.com/kubernetes/kubernetes/pull/85466), [@pjh](https://github.com/pjh))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "pjh",
-    "author_url": "https://github.com/pjh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85466",
-    "pr_number": 85466,
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85475": {
-    "commit": "97e34ee223c1a7e2c83399d789c8139d29be88aa",
-    "text": "fix azure file AuthorizationFailure",
-    "markdown": "fix azure file AuthorizationFailure ([#85475](https://github.com/kubernetes/kubernetes/pull/85475), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85475",
-    "pr_number": 85475,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85490": {
-    "commit": "7c87b5fb55ca096c007c8739d4657a5a4e29fb09",
-    "text": "kubectl set resources will no longer return an error if passed an empty change for a resource. \nkubectl set subject will no longer return an error if passed an empty change for a resource.",
-    "markdown": "kubectl set resources will no longer return an error if passed an empty change for a resource. \n  kubectl set subject will no longer return an error if passed an empty change for a resource. ([#85490](https://github.com/kubernetes/kubernetes/pull/85490), [@sallyom](https://github.com/sallyom))",
-    "author": "sallyom",
-    "author_url": "https://github.com/sallyom",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85490",
-    "pr_number": 85490,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85511": {
-    "commit": "92a14f4f7ef11c580961f0aebe491d0ff148e191",
-    "text": "Fix a bug in port-forward: named port not working with service",
-    "markdown": "Fix a bug in port-forward: named port not working with service ([#85511](https://github.com/kubernetes/kubernetes/pull/85511), [@oke-py](https://github.com/oke-py))",
-    "author": "oke-py",
-    "author_url": "https://github.com/oke-py",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85511",
-    "pr_number": 85511,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85512": {
-    "commit": "127c47caf49bec4a59a0500f99b1b1d4cdbf340e",
-    "text": "Remove cluster-monitoring addon",
-    "markdown": "Remove cluster-monitoring addon ([#85512](https://github.com/kubernetes/kubernetes/pull/85512), [@serathius](https://github.com/serathius))",
-    "author": "serathius",
-    "author_url": "https://github.com/serathius",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85512",
-    "pr_number": 85512,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["cleanup", "deprecation"],
-    "sigs": ["cluster-lifecycle", "instrumentation", "scalability", "testing"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85516": {
-    "commit": "1682a4a0b91edc64b42728a2b92aefe48d69ccdb",
-    "text": "kube-apiserver: fixed a conflict error encountered attempting to delete a pod with gracePeriodSeconds=0 and a resourceVersion precondition",
-    "markdown": "kube-apiserver: fixed a conflict error encountered attempting to delete a pod with gracePeriodSeconds=0 and a resourceVersion precondition ([#85516](https://github.com/kubernetes/kubernetes/pull/85516), [@michaelgugino](https://github.com/michaelgugino))",
-    "author": "michaelgugino",
-    "author_url": "https://github.com/michaelgugino",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85516",
-    "pr_number": 85516,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85517": {
-    "commit": "e1c711227691bd2b32519e86513bbad88b65c972",
-    "text": "add kube-proxy flags --ipvs-tcp-timeout, --ipvs-tcpfin-timeout, --ipvs-udp-timeout to configure IPVS connection timeouts.",
-    "markdown": "add kube-proxy flags --ipvs-tcp-timeout, --ipvs-tcpfin-timeout, --ipvs-udp-timeout to configure IPVS connection timeouts. ([#85517](https://github.com/kubernetes/kubernetes/pull/85517), [@andrewsykim](https://github.com/andrewsykim))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Network",
-    "author": "andrewsykim",
-    "author_url": "https://github.com/andrewsykim",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85517",
-    "pr_number": 85517,
-    "areas": ["ipvs", "kubeadm"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["cluster-lifecycle", "network"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85542": {
-    "commit": "541b4e4b395b5124f3517b4e62f3dc604016db11",
-    "text": "e2e-test-framework: add e2e test namespace dump if all tests succeed but the cleanup fails.",
-    "markdown": "e2e-test-framework: add e2e test namespace dump if all tests succeed but the cleanup fails. ([#85542](https://github.com/kubernetes/kubernetes/pull/85542), [@schrodit](https://github.com/schrodit))",
-    "author": "schrodit",
-    "author_url": "https://github.com/schrodit",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85542",
-    "pr_number": 85542,
-    "areas": ["e2e-test-framework", "test"],
-    "kinds": ["bug", "failing-test"],
-    "sigs": ["testing"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85543": {
-    "commit": "650c797d3faedcbd3ad60cad065cabf305bb49a1",
-    "text": "addons: elasticsearch discovery supports IPv6",
-    "markdown": "addons: elasticsearch discovery supports IPv6 ([#85543](https://github.com/kubernetes/kubernetes/pull/85543), [@SataQiu](https://github.com/SataQiu))",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85543",
-    "pr_number": 85543,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85571": {
-    "commit": "b42f21982c965630f82f053425ba067d818203f5",
-    "text": "kubectl/drain: add disable-eviction option.\nForce drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, and should be used with caution.",
-    "markdown": "kubectl/drain: add disable-eviction option.\n  Force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, and should be used with caution. ([#85571](https://github.com/kubernetes/kubernetes/pull/85571), [@michaelgugino](https://github.com/michaelgugino))\n\n  Courtesy of SIG CLI",
-    "author": "michaelgugino",
-    "author_url": "https://github.com/michaelgugino",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85571",
-    "pr_number": 85571,
-    "areas": ["kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85577": {
-    "commit": "93eb2e38a7800087f0ab2ae058cccd4dcceae8f9",
-    "text": "kubectl/drain: add skip-wait-for-delete-timeout option.\nIf pod DeletionTimestamp older than N seconds, skip waiting for the pod.  Seconds must be greater than 0 to skip.",
-    "markdown": "kubectl/drain: add skip-wait-for-delete-timeout option.\n  If pod DeletionTimestamp older than N seconds, skip waiting for the pod.  Seconds must be greater than 0 to skip. ([#85577](https://github.com/kubernetes/kubernetes/pull/85577), [@michaelgugino](https://github.com/michaelgugino))\n\n  Courtesy of SIG CLI",
-    "documentation": [
-      {
-        "description": "[Other doc]",
-        "url": "https://github.com/kubernetes/enhancements/issues/1339",
-        "type": "KEP"
-      }
-    ],
-    "author": "michaelgugino",
-    "author_url": "https://github.com/michaelgugino",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85577",
-    "pr_number": 85577,
-    "areas": ["kubectl"],
-    "kinds": ["feature"],
-    "sigs": ["cli"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85603": {
-    "commit": "e51caab73241b5c1598dfac5acbb2bde291a5e1f",
-    "text": "kubeadm: make sure images are pre-pulled even if a tag did not change but their contents changed",
-    "markdown": "kubeadm: make sure images are pre-pulled even if a tag did not change but their contents changed ([#85603](https://github.com/kubernetes/kubernetes/pull/85603), [@bart0sh](https://github.com/bart0sh))",
-    "author": "bart0sh",
-    "author_url": "https://github.com/bart0sh",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85603",
-    "pr_number": 85603,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85609": {
-    "commit": "7907c63fad1ccb7905d2d27c643387046ab0f4ad",
-    "text": "kubeadm now supports automatic calculations of dual-stack node cidr masks to kube-controller-manager.",
-    "markdown": "kubeadm now supports automatic calculations of dual-stack node cidr masks to kube-controller-manager. ([#85609](https://github.com/kubernetes/kubernetes/pull/85609), [@Arvinderpal](https://github.com/Arvinderpal))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "Arvinderpal",
-    "author_url": "https://github.com/Arvinderpal",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85609",
-    "pr_number": 85609,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85610": {
-    "commit": "82ee37f3e1ef2be3bfca8031e9babfc748084b4f",
-    "text": "Update Cluster Autoscaler to 1.17.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.17.0",
-    "markdown": "Update Cluster Autoscaler to 1.17.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.17.0 ([#85610](https://github.com/kubernetes/kubernetes/pull/85610), [@losipiuk](https://github.com/losipiuk))",
-    "author": "losipiuk",
-    "author_url": "https://github.com/losipiuk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85610",
-    "pr_number": 85610,
-    "kinds": ["bug"],
-    "sigs": ["autoscaling", "cluster-lifecycle"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85621": {
-    "commit": "eef4c00ae93bd51125c215918a7b1c632d298610",
-    "text": "None.",
-    "markdown": "None. ([#85621](https://github.com/kubernetes/kubernetes/pull/85621), [@chewong](https://github.com/chewong))",
-    "author": "chewong",
-    "author_url": "https://github.com/chewong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85621",
-    "pr_number": 85621,
-    "areas": ["test"],
-    "kinds": ["cleanup"],
-    "sigs": ["storage", "testing"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85639": {
-    "commit": "674695c544296c90e0a4fa0abdab71b3679a82c3",
-    "text": "kubeadm now errors out whenever a not supported component config version is supplied for the kubelet and kube-proxy",
-    "markdown": "kubeadm now errors out whenever a not supported component config version is supplied for the kubelet and kube-proxy ([#85639](https://github.com/kubernetes/kubernetes/pull/85639), [@rosti](https://github.com/rosti))",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-cluster-lifecycle/kubeadm/20190925-component-configs.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "rosti",
-    "author_url": "https://github.com/rosti",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85639",
-    "pr_number": 85639,
-    "areas": ["kubeadm"],
-    "kinds": ["design"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85675": {
-    "commit": "b920b388e90d32ea268b1b23142b65514fe6139e",
-    "text": "Fixed \"requested device X but found Y\" attach error on AWS.",
-    "markdown": "Fixed \"requested device X but found Y\" attach error on AWS. ([#85675](https://github.com/kubernetes/kubernetes/pull/85675), [@jsafrane](https://github.com/jsafrane))",
-    "author": "jsafrane",
-    "author_url": "https://github.com/jsafrane",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85675",
-    "pr_number": 85675,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85722": {
-    "commit": "1d343c8a0ce3e4d4845f7c34d06a1e2f25c975b7",
-    "text": "Filter published OpenAPI schema by making nullable, required fields non-required in order to avoid kubectl to wrongly reject null values.",
-    "markdown": "Filter published OpenAPI schema by making nullable, required fields non-required in order to avoid kubectl to wrongly reject null values. ([#85722](https://github.com/kubernetes/kubernetes/pull/85722), [@sttts](https://github.com/sttts))",
-    "author": "sttts",
-    "author_url": "https://github.com/sttts",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85722",
-    "pr_number": 85722,
-    "areas": ["custom-resources"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85739": {
-    "commit": "220e2ceba0784fe230d590d58e26d87cbf05af13",
-    "text": "Update go-winio module version from 0.4.11 to 0.4.14",
-    "markdown": "Update go-winio module version from 0.4.11 to 0.4.14 ([#85739](https://github.com/kubernetes/kubernetes/pull/85739), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85739",
-    "pr_number": 85739,
-    "areas": ["dependency"],
-    "kinds": ["bug"],
-    "sigs": ["windows"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85745": {
-    "commit": "3769de9401474ea22c0ddbacde896911ddd87700",
-    "text": "kubeadm: uses the apiserver AdvertiseAddress IP family to choose the etcd endpoint IP family for non external etcd clusters",
-    "markdown": "kubeadm: uses the apiserver AdvertiseAddress IP family to choose the etcd endpoint IP family for non external etcd clusters ([#85745](https://github.com/kubernetes/kubernetes/pull/85745), [@aojea](https://github.com/aojea))",
-    "author": "aojea",
-    "author_url": "https://github.com/aojea",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85745",
-    "pr_number": 85745,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85747": {
-    "commit": "0ab15832400c762c7203721d551d2072e2951c6d",
-    "text": "Fixes wrong validation result of NetworkPolicy PolicyTypes",
-    "markdown": "Fixes wrong validation result of NetworkPolicy PolicyTypes ([#85747](https://github.com/kubernetes/kubernetes/pull/85747), [@tnqn](https://github.com/tnqn))",
-    "author": "tnqn",
-    "author_url": "https://github.com/tnqn",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85747",
-    "pr_number": 85747,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85763": {
-    "commit": "1ca289dfc11547af7b1e7aa6c6fd09972260a7c6",
-    "text": "kubeadm: retry `kubeadm-config` ConfigMap creation or mutation if the apiserver is not responding. This will improve resiliency when joining new control plane nodes.",
-    "markdown": "kubeadm: retry `kubeadm-config` ConfigMap creation or mutation if the apiserver is not responding. This will improve resiliency when joining new control plane nodes. ([#85763](https://github.com/kubernetes/kubernetes/pull/85763), [@ereslibre](https://github.com/ereslibre))",
-    "author": "ereslibre",
-    "author_url": "https://github.com/ereslibre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85763",
-    "pr_number": 85763,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85771": {
-    "commit": "d0e901807d4c11e725b7b3ae95d312a13d5dd7f8",
-    "text": "kubernetes will try to acquire the iptables lock every 100 msec during 5 seconds instead of every second. This specially useful for environments using kube-proxy in iptables mode with a high churn rate of services.",
-    "markdown": "kubernetes will try to acquire the iptables lock every 100 msec during 5 seconds instead of every second. This specially useful for environments using kube-proxy in iptables mode with a high churn rate of services. ([#85771](https://github.com/kubernetes/kubernetes/pull/85771), [@aojea](https://github.com/aojea))",
-    "author": "aojea",
-    "author_url": "https://github.com/aojea",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85771",
-    "pr_number": 85771,
-    "kinds": ["flake"],
-    "sigs": ["network"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85789": {
-    "commit": "6db550d1db70e389463930bccfb4855f7369b815",
-    "text": "OwnerReferences []OwnerReferencematches \nso nil len(OwnerReferences) ==0 \n\nchange:\n if lease.OwnerReferences == nil || len(lease.OwnerReferences) == 0 ==\u003eif len(lease.OwnerReferences) == 0",
-    "markdown": "OwnerReferences []OwnerReferencematches \n  so nil len(OwnerReferences) ==0 \n  \n  change:\n   if lease.OwnerReferences == nil || len(lease.OwnerReferences) == 0 ==\u003eif len(lease.OwnerReferences) == 0 ([#85789](https://github.com/kubernetes/kubernetes/pull/85789), [@ZP-AlwaysWin](https://github.com/ZP-AlwaysWin))",
-    "author": "ZP-AlwaysWin",
-    "author_url": "https://github.com/ZP-AlwaysWin",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85789",
-    "pr_number": 85789,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85793": {
-    "commit": "85b5a046f7de104281225485797232d6cdd71389",
-    "text": "Fixed issue with addon-resizer using deprecated extensions APIs",
-    "markdown": "Fixed issue with addon-resizer using deprecated extensions APIs ([#85793](https://github.com/kubernetes/kubernetes/pull/85793), [@bskiba](https://github.com/bskiba))",
-    "author": "bskiba",
-    "author_url": "https://github.com/bskiba",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85793",
-    "pr_number": 85793,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85810": {
-    "commit": "19ee1ea9fa61b2a3fe4d86438bef5203bfe1a756",
-    "text": "Resolved regression in admission, authentication, and authorization webhook performance in v1.17.0-rc.1",
-    "markdown": "Resolved regression in admission, authentication, and authorization webhook performance in v1.17.0-rc.1 ([#85810](https://github.com/kubernetes/kubernetes/pull/85810), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85810",
-    "pr_number": 85810,
-    "areas": ["apiserver", "test"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "testing"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85817": {
-    "commit": "6e0ce9fb2ff8f26d1becafedfc732d603363b9a7",
-    "text": "kubeadm: Forward cluster name to the controller-manager arguments",
-    "markdown": "kubeadm: Forward cluster name to the controller-manager arguments ([#85817](https://github.com/kubernetes/kubernetes/pull/85817), [@ereslibre](https://github.com/ereslibre))",
-    "author": "ereslibre",
-    "author_url": "https://github.com/ereslibre",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85817",
-    "pr_number": 85817,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85837": {
-    "commit": "7b792c38e0c9b66413d790236c80367df3094484",
-    "text": "allow for configuration of CoreDNS replica count",
-    "markdown": "allow for configuration of CoreDNS replica count ([#85837](https://github.com/kubernetes/kubernetes/pull/85837), [@pickledrick](https://github.com/pickledrick))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "pickledrick",
-    "author_url": "https://github.com/pickledrick",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85837",
-    "pr_number": 85837,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85857": {
-    "commit": "2fbe432d2347d7f808054f92f2b146e8a7dd2de8",
-    "text": "During initialization phase (preflight), kubeadm now verifies the presence of the conntrack executable",
-    "markdown": "During initialization phase (preflight), kubeadm now verifies the presence of the conntrack executable ([#85857](https://github.com/kubernetes/kubernetes/pull/85857), [@hnanni](https://github.com/hnanni))",
-    "author": "hnanni",
-    "author_url": "https://github.com/hnanni",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85857",
-    "pr_number": 85857,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85874": {
-    "commit": "398e2bcc734f04686ff89fdd9f5b47cf12cd525b",
-    "text": "- Renamed Kubelet metric certificate_manager_server_expiration_seconds to certificate_manager_server_ttl_seconds and changed to report the second until expiration at read time rather than absolute time of expiry.\n- Improved accuracy of Kubelet metric rest_client_exec_plugin_ttl_seconds.",
-    "markdown": "- Renamed Kubelet metric certificate_manager_server_expiration_seconds to certificate_manager_server_ttl_seconds and changed to report the second until expiration at read time rather than absolute time of expiry.\n  - Improved accuracy of Kubelet metric rest_client_exec_plugin_ttl_seconds. ([#85874](https://github.com/kubernetes/kubernetes/pull/85874), [@sambdavidson](https://github.com/sambdavidson))",
-    "author": "sambdavidson",
-    "author_url": "https://github.com/sambdavidson",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85874",
-    "pr_number": 85874,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "auth", "cluster-lifecycle", "instrumentation", "node"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85885": {
-    "commit": "50437b4c5da29baba1f8aeef201c2b7736cd42d2",
-    "text": "VMSS cache is added so that less chances of VMSS GET throttling",
-    "markdown": "VMSS cache is added so that less chances of VMSS GET throttling ([#85885](https://github.com/kubernetes/kubernetes/pull/85885), [@nilo19](https://github.com/nilo19))",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85885",
-    "pr_number": 85885,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85903": {
-    "commit": "6a919f2b54bc3fbc047250f0aa8c423f5209c2b5",
-    "text": "kube-apiserver no longer serves the following deprecated APIs:\n- All resources under `apps/v1beta1` and `apps/v1beta2` - use `apps/v1` instead\n- `daemonsets`, `deployments`, `replicasets` resources under `extensions/v1beta1` - use `apps/v1` instead\n- `networkpolicies` resources under `extensions/v1beta1` - use `networking.k8s.io/v1` instead\n- `podsecuritypolicies` resources under `extensions/v1beta1` - use `policy/v1beta1` instead",
-    "markdown": "kube-apiserver no longer serves the following deprecated APIs:\n  - All resources under `apps/v1beta1` and `apps/v1beta2` - use `apps/v1` instead\n  - `daemonsets`, `deployments`, `replicasets` resources under `extensions/v1beta1` - use `apps/v1` instead\n  - `networkpolicies` resources under `extensions/v1beta1` - use `networking.k8s.io/v1` instead\n  - `podsecuritypolicies` resources under `extensions/v1beta1` - use `policy/v1beta1` instead ([#85903](https://github.com/kubernetes/kubernetes/pull/85903), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85903",
-    "pr_number": 85903,
-    "areas": ["apiserver", "test"],
-    "kinds": ["api-change", "cleanup"],
-    "sigs": ["api-machinery", "apps", "cluster-lifecycle", "instrumentation", "testing"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "85990": {
-    "commit": "b5040be796b71f07de2f0be782d14e9b24b018b7",
-    "text": "Fix LoadBalancer rule checking so that no unexpected LoadBalancer updates are made",
-    "markdown": "Fix LoadBalancer rule checking so that no unexpected LoadBalancer updates are made ([#85990](https://github.com/kubernetes/kubernetes/pull/85990), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85990",
-    "pr_number": 85990,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "86016": {
-    "commit": "078344c12524adfe1f6f742e0d5a21016d65e64b",
-    "text": "Fixes kube-proxy when EndpointSlice feature gate is enabled on Windows.",
-    "markdown": "Fixes kube-proxy when EndpointSlice feature gate is enabled on Windows. ([#86016](https://github.com/kubernetes/kubernetes/pull/86016), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86016",
-    "pr_number": 86016,
-    "kinds": ["bug"],
-    "sigs": ["auth", "network"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "86022": {
-    "commit": "d842c194c21f86ddcecae303ce9d8dcce882a18c",
-    "text": "Fixed an issue that the scheduler only returns the first failure reason.",
-    "markdown": "Fixed an issue that the scheduler only returns the first failure reason. ([#86022](https://github.com/kubernetes/kubernetes/pull/86022), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86022",
-    "pr_number": 86022,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "86078": {
-    "commit": "939da5e9cf7a041fd5cdf992449372e7b9833162",
-    "text": "Make error message and service event message more clear",
-    "markdown": "Make error message and service event message more clear ([#86078](https://github.com/kubernetes/kubernetes/pull/86078), [@feiskyer](https://github.com/feiskyer))",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86078",
-    "pr_number": 86078,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "86092": {
-    "commit": "442107b6b96a980e4d26c30a176071092748414e",
-    "text": "kube-proxy no longer modifies shared EndpointSlices.",
-    "markdown": "kube-proxy no longer modifies shared EndpointSlices. ([#86092](https://github.com/kubernetes/kubernetes/pull/86092), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86092",
-    "pr_number": 86092,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "86168": {
-    "commit": "0d58709016f3358f34a52c12087d707a64cc6cb0",
-    "text": "Resolves performance regression in `kubectl get all` and in client-go discovery clients constructed using `NewDiscoveryClientForConfig` or `NewDiscoveryClientForConfigOrDie`.",
-    "markdown": "Resolves performance regression in `kubectl get all` and in client-go discovery clients constructed using `NewDiscoveryClientForConfig` or `NewDiscoveryClientForConfigOrDie`. ([#86168](https://github.com/kubernetes/kubernetes/pull/86168), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86168",
-    "pr_number": 86168,
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "86216": {
-    "commit": "e622579b143730b89d471ecd4e5f6761e9c37d86",
-    "text": "kubeadm: fix potential panic when executing \"kubeadm reset\" with a corrupted kubelet.conf file",
-    "markdown": "kubeadm: fix potential panic when executing \"kubeadm reset\" with a corrupted kubelet.conf file ([#86216](https://github.com/kubernetes/kubernetes/pull/86216), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86216",
-    "pr_number": 86216,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "86266": {
-    "commit": "f06a3800ed7819fa68aa30054b4cd52619a4cbde",
-    "text": "azure cloud provider cache TTL is configurable, list of the azure cloud provider is as following:\n- \"availabilitySetNodesCacheTTLInSeconds\"\n- \"vmssCacheTTLInSeconds\"\n- \"vmssVirtualMachinesCacheTTLInSeconds\"\n- \"vmCacheTTLInSeconds\"\n- \"loadBalancerCacheTTLInSeconds\"\n- \"nsgCacheTTLInSeconds\"\n- \"routeTableCacheTTLInSeconds\"",
-    "markdown": "azure cloud provider cache TTL is configurable, list of the azure cloud provider is as following:\n  - \"availabilitySetNodesCacheTTLInSeconds\"\n  - \"vmssCacheTTLInSeconds\"\n  - \"vmssVirtualMachinesCacheTTLInSeconds\"\n  - \"vmCacheTTLInSeconds\"\n  - \"loadBalancerCacheTTLInSeconds\"\n  - \"nsgCacheTTLInSeconds\"\n  - \"routeTableCacheTTLInSeconds\" ([#86266](https://github.com/kubernetes/kubernetes/pull/86266), [@zqingqing1](https://github.com/zqingqing1))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "zqingqing1",
-    "author_url": "https://github.com/zqingqing1",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86266",
-    "pr_number": 86266,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "86277": {
-    "commit": "a57a8f33b323cd002e76c664bf0f91f71df7c927",
-    "text": "Fixed a panic in the kubelet cleaning up pod volumes",
-    "markdown": "Fixed a panic in the kubelet cleaning up pod volumes ([#86277](https://github.com/kubernetes/kubernetes/pull/86277), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86277",
-    "pr_number": 86277,
-    "kinds": ["bug"],
-    "sigs": ["storage"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "86320": {
-    "commit": "eee9dee7b9c0ad926c8619b211902f118feff2fa",
-    "text": "Fixes an issue with kubelet-reported pod status on deleted/recreated pods.",
-    "markdown": "Fixes an issue with kubelet-reported pod status on deleted/recreated pods. ([#86320](https://github.com/kubernetes/kubernetes/pull/86320), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86320",
-    "pr_number": 86320,
-    "areas": ["kubelet"],
-    "kinds": ["bug", "flake"],
-    "sigs": ["node"],
-    "release_version": "1.18.0-alpha.1"
-  },
-  "86329": {
-    "commit": "4a62b3ac6dc7937ea798f2163cb3bb1f1a165f20",
-    "text": "Changed core_pattern on COS nodes to be an absolute path.",
-    "markdown": "Changed core_pattern on COS nodes to be an absolute path. ([#86329](https://github.com/kubernetes/kubernetes/pull/86329), [@mml](https://github.com/mml))",
-    "author": "mml",
-    "author_url": "https://github.com/mml",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86329",
-    "pr_number": 86329,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle", "node"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.1"
-  },
-  "86351": {
-    "commit": "ac4079a5cc22ced66c868f6fd26f8e2497765e51",
-    "text": "fix: azure data disk should use same key as os disk by default",
-    "markdown": "fix: azure data disk should use same key as os disk by default ([#86351](https://github.com/kubernetes/kubernetes/pull/86351), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86351",
-    "pr_number": 86351,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.18.0-alpha.1"
-  }
-}
diff --git a/src/assets/release-notes-1.18.0-alpha.2.json b/src/assets/release-notes-1.18.0-alpha.2.json
deleted file mode 100644
index cc371868..00000000
--- a/src/assets/release-notes-1.18.0-alpha.2.json
+++ /dev/null
@@ -1,814 +0,0 @@
-{
-  "73363": {
-    "commit": "67647ed5a379ea72d7db7b5461e9802aa1debdbf",
-    "text": "Minimize AWS NLB health check timeout when externalTrafficPolicy set to Local",
-    "markdown": "Minimize AWS NLB health check timeout when externalTrafficPolicy set to Local ([#73363](https://github.com/kubernetes/kubernetes/pull/73363), [@kellycampbell](https://github.com/kellycampbell))",
-    "author": "kellycampbell",
-    "author_url": "https://github.com/kellycampbell",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/73363",
-    "pr_number": 73363,
-    "areas": ["cloudprovider", "provider/aws"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "76496": {
-    "commit": "f4db8212be53c69a27d893d6a4111422fbce8008",
-    "text": "The following deprecated metrics are removed, please convert to the corresponding metrics:\n1. The following replacement metrics are available from v1.14.0:\n- `rest_client_request_latency_seconds` -\u003e `rest_client_request_duration_seconds`\n- `scheduler_scheduling_latency_seconds` -\u003e `scheduler_scheduling_duration_seconds `\n- `docker_operations` -\u003e `docker_operations_total`\n- `docker_operations_latency_microseconds` -\u003e `docker_operations_duration_seconds`\n- `docker_operations_errors` -\u003e `docker_operations_errors_total`\n- `docker_operations_timeout` -\u003e `docker_operations_timeout_total`\n- `network_plugin_operations_latency_microseconds` -\u003e `network_plugin_operations_duration_seconds`\n- `kubelet_pod_worker_latency_microseconds` -\u003e `kubelet_pod_worker_duration_seconds`\n- `kubelet_pod_start_latency_microseconds` -\u003e `kubelet_pod_start_duration_seconds`\n- `kubelet_cgroup_manager_latency_microseconds` -\u003e `kubelet_cgroup_manager_duration_seconds`\n- `kubelet_pod_worker_start_latency_microseconds` -\u003e `kubelet_pod_worker_start_duration_seconds`\n- `kubelet_pleg_relist_latency_microseconds` -\u003e `kubelet_pleg_relist_duration_seconds`\n- `kubelet_pleg_relist_interval_microseconds` -\u003e `kubelet_pleg_relist_interval_seconds`\n- `kubelet_eviction_stats_age_microseconds` -\u003e `kubelet_eviction_stats_age_seconds`\n- `kubelet_runtime_operations` -\u003e `kubelet_runtime_operations_total`\n- `kubelet_runtime_operations_latency_microseconds` -\u003e `kubelet_runtime_operations_duration_seconds`\n- `kubelet_runtime_operations_errors` -\u003e `kubelet_runtime_operations_errors_total`\n- `kubelet_device_plugin_registration_count` -\u003e `kubelet_device_plugin_registration_total`\n- `kubelet_device_plugin_alloc_latency_microseconds` -\u003e `kubelet_device_plugin_alloc_duration_seconds`\n- `scheduler_e2e_scheduling_latency_microseconds` -\u003e `scheduler_e2e_scheduling_duration_seconds`\n- `scheduler_scheduling_algorithm_latency_microseconds` -\u003e `scheduler_scheduling_algorithm_duration_seconds`\n- `scheduler_scheduling_algorithm_predicate_evaluation` -\u003e `scheduler_scheduling_algorithm_predicate_evaluation_seconds`\n- `scheduler_scheduling_algorithm_priority_evaluation` -\u003e `scheduler_scheduling_algorithm_priority_evaluation_seconds`\n- `scheduler_scheduling_algorithm_preemption_evaluation` -\u003e `scheduler_scheduling_algorithm_preemption_evaluation_seconds`\n- `scheduler_binding_latency_microseconds` -\u003e `scheduler_binding_duration_seconds`\n- `kubeproxy_sync_proxy_rules_latency_microseconds` -\u003e `kubeproxy_sync_proxy_rules_duration_seconds`\n- `apiserver_request_latencies` -\u003e `apiserver_request_duration_seconds`\n- `apiserver_dropped_requests` -\u003e `apiserver_dropped_requests_total`\n- `etcd_request_latencies_summary` -\u003e `etcd_request_duration_seconds`\n- `apiserver_storage_transformation_latencies_microseconds ` -\u003e `apiserver_storage_transformation_duration_seconds`\n- `apiserver_storage_data_key_generation_latencies_microseconds` -\u003e `apiserver_storage_data_key_generation_duration_seconds`\n- `apiserver_request_count` -\u003e `apiserver_request_total`\n- `apiserver_request_latencies_summary`\n2. The following replacement metrics are available from v1.15.0:\n- `apiserver_storage_transformation_failures_total` -\u003e `apiserver_storage_transformation_operations_total`",
-    "markdown": "The following deprecated metrics are removed, please convert to the corresponding metrics:\n  1. The following replacement metrics are available from v1.14.0:\n  - `rest_client_request_latency_seconds` -\u003e `rest_client_request_duration_seconds`\n  - `scheduler_scheduling_latency_seconds` -\u003e `scheduler_scheduling_duration_seconds `\n  - `docker_operations` -\u003e `docker_operations_total`\n  - `docker_operations_latency_microseconds` -\u003e `docker_operations_duration_seconds`\n  - `docker_operations_errors` -\u003e `docker_operations_errors_total`\n  - `docker_operations_timeout` -\u003e `docker_operations_timeout_total`\n  - `network_plugin_operations_latency_microseconds` -\u003e `network_plugin_operations_duration_seconds`\n  - `kubelet_pod_worker_latency_microseconds` -\u003e `kubelet_pod_worker_duration_seconds`\n  - `kubelet_pod_start_latency_microseconds` -\u003e `kubelet_pod_start_duration_seconds`\n  - `kubelet_cgroup_manager_latency_microseconds` -\u003e `kubelet_cgroup_manager_duration_seconds`\n  - `kubelet_pod_worker_start_latency_microseconds` -\u003e `kubelet_pod_worker_start_duration_seconds`\n  - `kubelet_pleg_relist_latency_microseconds` -\u003e `kubelet_pleg_relist_duration_seconds`\n  - `kubelet_pleg_relist_interval_microseconds` -\u003e `kubelet_pleg_relist_interval_seconds`\n  - `kubelet_eviction_stats_age_microseconds` -\u003e `kubelet_eviction_stats_age_seconds`\n  - `kubelet_runtime_operations` -\u003e `kubelet_runtime_operations_total`\n  - `kubelet_runtime_operations_latency_microseconds` -\u003e `kubelet_runtime_operations_duration_seconds`\n  - `kubelet_runtime_operations_errors` -\u003e `kubelet_runtime_operations_errors_total`\n  - `kubelet_device_plugin_registration_count` -\u003e `kubelet_device_plugin_registration_total`\n  - `kubelet_device_plugin_alloc_latency_microseconds` -\u003e `kubelet_device_plugin_alloc_duration_seconds`\n  - `scheduler_e2e_scheduling_latency_microseconds` -\u003e `scheduler_e2e_scheduling_duration_seconds`\n  - `scheduler_scheduling_algorithm_latency_microseconds` -\u003e `scheduler_scheduling_algorithm_duration_seconds`\n  - `scheduler_scheduling_algorithm_predicate_evaluation` -\u003e `scheduler_scheduling_algorithm_predicate_evaluation_seconds`\n  - `scheduler_scheduling_algorithm_priority_evaluation` -\u003e `scheduler_scheduling_algorithm_priority_evaluation_seconds`\n  - `scheduler_scheduling_algorithm_preemption_evaluation` -\u003e `scheduler_scheduling_algorithm_preemption_evaluation_seconds`\n  - `scheduler_binding_latency_microseconds` -\u003e `scheduler_binding_duration_seconds`\n  - `kubeproxy_sync_proxy_rules_latency_microseconds` -\u003e `kubeproxy_sync_proxy_rules_duration_seconds`\n  - `apiserver_request_latencies` -\u003e `apiserver_request_duration_seconds`\n  - `apiserver_dropped_requests` -\u003e `apiserver_dropped_requests_total`\n  - `etcd_request_latencies_summary` -\u003e `etcd_request_duration_seconds`\n  - `apiserver_storage_transformation_latencies_microseconds ` -\u003e `apiserver_storage_transformation_duration_seconds`\n  - `apiserver_storage_data_key_generation_latencies_microseconds` -\u003e `apiserver_storage_data_key_generation_duration_seconds`\n  - `apiserver_request_count` -\u003e `apiserver_request_total`\n  - `apiserver_request_latencies_summary`\n  2. The following replacement metrics are available from v1.15.0:\n  - `apiserver_storage_transformation_failures_total` -\u003e `apiserver_storage_transformation_operations_total` ([#76496](https://github.com/kubernetes/kubernetes/pull/76496), [@danielqsj](https://github.com/danielqsj))",
-    "author": "danielqsj",
-    "author_url": "https://github.com/danielqsj",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/76496",
-    "pr_number": 76496,
-    "areas": ["apiserver", "ipvs", "kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": [
-      "api-machinery",
-      "cluster-lifecycle",
-      "instrumentation",
-      "network",
-      "node",
-      "scheduling"
-    ],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "82462": {
-    "commit": "5373fa3f593a12309ff0435858991dd117bdb6ca",
-    "text": "kube-proxy: Added dual-stack IPv4/IPv6 support to the iptables proxier.",
-    "markdown": "kube-proxy: Added dual-stack IPv4/IPv6 support to the iptables proxier. ([#82462](https://github.com/kubernetes/kubernetes/pull/82462), [@vllry](https://github.com/vllry))\n\n  Courtesy of SIG Network",
-    "author": "vllry",
-    "author_url": "https://github.com/vllry",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/82462",
-    "pr_number": 82462,
-    "areas": ["ipvs"],
-    "kinds": ["feature"],
-    "sigs": ["network"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "83358": {
-    "commit": "0c026656fe1c697594a756e543578ff3eab98060",
-    "text": "Bind kube-dns containers to linux nodes to avoid Windows scheduling",
-    "markdown": "Bind kube-dns containers to linux nodes to avoid Windows scheduling ([#83358](https://github.com/kubernetes/kubernetes/pull/83358), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83358",
-    "pr_number": 83358,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "83364": {
-    "commit": "fc98aa54465dbb31e2260cc51fb753c8eae55a9e",
-    "text": "Bind dns-horizontal containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes",
-    "markdown": "Bind dns-horizontal containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes ([#83364](https://github.com/kubernetes/kubernetes/pull/83364), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83364",
-    "pr_number": 83364,
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle", "windows"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "83437": {
-    "commit": "9fa1e00be9f4816e18d463f8125a7f20c0d4aee1",
-    "text": "Promote StartupProbe to beta for 1.18 release",
-    "markdown": "Promote StartupProbe to beta for 1.18 release ([#83437](https://github.com/kubernetes/kubernetes/pull/83437), [@matthyx](https://github.com/matthyx))\n\n  Courtesy of SIG Node, SIG Scalability, and SIG Testing",
-    "author": "matthyx",
-    "author_url": "https://github.com/matthyx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/83437",
-    "pr_number": 83437,
-    "areas": ["test"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["node", "scalability", "testing"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "84163": {
-    "commit": "daaa0b9c6ba2dbf37cc0ae4464ed8b35521c7b62",
-    "text": "Bug fixes:\nMake sure we include latest packages node \u0026#35;351 (@caseydavenport)",
-    "markdown": "Bug fixes:\n  Make sure we include latest packages node \u0026#35;351 (@caseydavenport) ([#84163](https://github.com/kubernetes/kubernetes/pull/84163), [@david-tigera](https://github.com/david-tigera))",
-    "author": "david-tigera",
-    "author_url": "https://github.com/david-tigera",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84163",
-    "pr_number": 84163,
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "84265": {
-    "commit": "c9003a268dff6700506929b247847341ea4d5b33",
-    "text": "Fixed the following \n-  AWS Cloud Provider attempts to delete LoadBalancer security group it didn’t provision\n-  AWS Cloud Provider creates default LoadBalancer security group even if annotation [service.beta.kubernetes.io/aws-load-balancer-security-groups] is present",
-    "markdown": "Fixed the following \n  -  AWS Cloud Provider attempts to delete LoadBalancer security group it didn’t provision\n  -  AWS Cloud Provider creates default LoadBalancer security group even if annotation [service.beta.kubernetes.io/aws-load-balancer-security-groups] is present ([#84265](https://github.com/kubernetes/kubernetes/pull/84265), [@bhagwat070919](https://github.com/bhagwat070919))",
-    "author": "bhagwat070919",
-    "author_url": "https://github.com/bhagwat070919",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/84265",
-    "pr_number": 84265,
-    "areas": ["cloudprovider", "provider/aws"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "85281": {
-    "commit": "d9bf22f8f7df184cf427114e8ad1fef12474364f",
-    "text": "New flag `--show-hidden-metrics-for-version` in kube-controller-manager can be used to show all hidden metrics that deprecated in the previous minor release.",
-    "markdown": "New flag `--show-hidden-metrics-for-version` in kube-controller-manager can be used to show all hidden metrics that deprecated in the previous minor release. ([#85281](https://github.com/kubernetes/kubernetes/pull/85281), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG API Machinery",
-    "documentation": [
-      {
-        "description": "[KEP]",
-        "url": "https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md",
-        "type": "KEP"
-      }
-    ],
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85281",
-    "pr_number": 85281,
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "85283": {
-    "commit": "dd77283426bd3249004472efcbb3257744448161",
-    "text": "Cleaned up the output from `kubectl describe CSINode \u003cname\u003e`.",
-    "markdown": "Cleaned up the output from `kubectl describe CSINode \u003cname\u003e`. ([#85283](https://github.com/kubernetes/kubernetes/pull/85283), [@huffmanca](https://github.com/huffmanca))",
-    "author": "huffmanca",
-    "author_url": "https://github.com/huffmanca",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85283",
-    "pr_number": 85283,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli", "storage"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "85308": {
-    "commit": "a26fcf5278a7f643c3aa1dce9be60a3240bb1a72",
-    "text": "If a serving certificates param specifies a name that is an IP for an SNI certificate, it will have priority for replying to server connections.",
-    "markdown": "If a serving certificates param specifies a name that is an IP for an SNI certificate, it will have priority for replying to server connections. ([#85308](https://github.com/kubernetes/kubernetes/pull/85308), [@deads2k](https://github.com/deads2k))",
-    "author": "deads2k",
-    "author_url": "https://github.com/deads2k",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85308",
-    "pr_number": 85308,
-    "areas": ["apiserver", "dependency"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "85458": {
-    "commit": "7f2128757b1aa2e83320d1f635fe8b2e7028542a",
-    "text": "Improved yaml parsing performance",
-    "markdown": "Improved yaml parsing performance ([#85458](https://github.com/kubernetes/kubernetes/pull/85458), [@cjcullen](https://github.com/cjcullen))",
-    "author": "cjcullen",
-    "author_url": "https://github.com/cjcullen",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85458",
-    "pr_number": 85458,
-    "areas": ["apiserver", "cloudprovider", "code-generation", "dependency", "kubectl"],
-    "kinds": ["bug"],
-    "sigs": [
-      "api-machinery",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "node"
-    ],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "85613": {
-    "commit": "05209312b74eac15f0a80577cbbdcb11bce9b8ad",
-    "text": "`kubectl apply  -f \u003cfile\u003e --prune -n \u003cnamespace\u003e` should prune all resources not defined in the file in the cli specified namespace.",
-    "markdown": "`kubectl apply  -f \u003cfile\u003e --prune -n \u003cnamespace\u003e` should prune all resources not defined in the file in the cli specified namespace. ([#85613](https://github.com/kubernetes/kubernetes/pull/85613), [@MartinKaburu](https://github.com/MartinKaburu))",
-    "author": "MartinKaburu",
-    "author_url": "https://github.com/MartinKaburu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85613",
-    "pr_number": 85613,
-    "areas": ["kubectl"],
-    "kinds": ["bug"],
-    "sigs": ["cli"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "85703": {
-    "commit": "d4ba10ec808aec7c3529b4b216d428e024861225",
-    "text": "Fix EndpointSlice controller race condition and ensure that it handles external changes to EndpointSlices.",
-    "markdown": "Fix EndpointSlice controller race condition and ensure that it handles external changes to EndpointSlices. ([#85703](https://github.com/kubernetes/kubernetes/pull/85703), [@robscott](https://github.com/robscott))",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85703",
-    "pr_number": 85703,
-    "kinds": ["bug"],
-    "sigs": ["apps", "network"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "85836": {
-    "commit": "8727eef79c679bf40ff0cfa3318dadbc5fd50e24",
-    "text": "Add instructions about how to bring up e2e test cluster",
-    "markdown": "Add instructions about how to bring up e2e test cluster ([#85836](https://github.com/kubernetes/kubernetes/pull/85836), [@YangLu1031](https://github.com/YangLu1031))",
-    "author": "YangLu1031",
-    "author_url": "https://github.com/YangLu1031",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85836",
-    "pr_number": 85836,
-    "kinds": ["documentation"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "85850": {
-    "commit": "61d36e4a43b831f960b190b81d371cb33b5f20d1",
-    "text": "You can now pass \"--node-ip ::\" to kubelet to indicate that it should autodetect an IPv6 address to use as the node's primary address.",
-    "markdown": "You can now pass \"--node-ip ::\" to kubelet to indicate that it should autodetect an IPv6 address to use as the node's primary address. ([#85850](https://github.com/kubernetes/kubernetes/pull/85850), [@danwinship](https://github.com/danwinship))\n\n  Courtesy of SIG Cloud Provider, SIG Network, and SIG Node",
-    "author": "danwinship",
-    "author_url": "https://github.com/danwinship",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85850",
-    "pr_number": 85850,
-    "areas": ["ipv6", "kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider", "network", "node"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "85879": {
-    "commit": "814fc34cde8c51ffc83870e18ad703deece63af7",
-    "text": "hollow-node use fake CRI",
-    "markdown": "hollow-node use fake CRI ([#85879](https://github.com/kubernetes/kubernetes/pull/85879), [@gongguan](https://github.com/gongguan))\n\n  Courtesy of SIG Node, and SIG Scalability",
-    "author": "gongguan",
-    "author_url": "https://github.com/gongguan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85879",
-    "pr_number": 85879,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node", "scalability"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "85944": {
-    "commit": "a233e0115bb56e014f812365f583704075accc7b",
-    "text": "elasticsearch supports automatically setting the advertise address",
-    "markdown": "elasticsearch supports automatically setting the advertise address ([#85944](https://github.com/kubernetes/kubernetes/pull/85944), [@SataQiu](https://github.com/SataQiu))\n\n  Courtesy of SIG Cluster Lifecycle, and SIG Instrumentation",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85944",
-    "pr_number": 85944,
-    "kinds": ["bug", "feature"],
-    "sigs": ["cluster-lifecycle", "instrumentation"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "85975": {
-    "commit": "6f61072606155b070b2f79850b008685df3a74ff",
-    "text": "Fix nil pointer dereference in azure cloud provider",
-    "markdown": "Fix nil pointer dereference in azure cloud provider ([#85975](https://github.com/kubernetes/kubernetes/pull/85975), [@ldx](https://github.com/ldx))",
-    "author": "ldx",
-    "author_url": "https://github.com/ldx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85975",
-    "pr_number": 85975,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "85993": {
-    "commit": "7a50fdb2a635f4f3219f6411fa42d89c65a77800",
-    "text": "fix a regression in kubenet that prevent pods to obtain ip addresses",
-    "markdown": "fix a regression in kubenet that prevent pods to obtain ip addresses ([#85993](https://github.com/kubernetes/kubernetes/pull/85993), [@chendotjs](https://github.com/chendotjs))",
-    "author": "chendotjs",
-    "author_url": "https://github.com/chendotjs",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/85993",
-    "pr_number": 85993,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["network", "node"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86137": {
-    "commit": "967b40f0513358cc03021d0b27cd5d85c62a95d0",
-    "text": "EndpointSlices will now be enabled by default. A new `EndpointSliceProxying` feature gate determines if kube-proxy will use EndpointSlices, this is disabled by default.",
-    "markdown": "EndpointSlices will now be enabled by default. A new `EndpointSliceProxying` feature gate determines if kube-proxy will use EndpointSlices, this is disabled by default. ([#86137](https://github.com/kubernetes/kubernetes/pull/86137), [@robscott](https://github.com/robscott))\n\n  Courtesy of SIG Auth, and SIG Network",
-    "author": "robscott",
-    "author_url": "https://github.com/robscott",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86137",
-    "pr_number": 86137,
-    "areas": ["ipvs"],
-    "kinds": ["feature"],
-    "sigs": ["auth", "network"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86167": {
-    "commit": "03e90b80ceebc4197eae7cc6b95a59623bd9a51f",
-    "text": "change CounterVec to Counter about  PLEGDiscardEvent",
-    "markdown": "change CounterVec to Counter about  PLEGDiscardEvent ([#86167](https://github.com/kubernetes/kubernetes/pull/86167), [@yiyang5055](https://github.com/yiyang5055))",
-    "author": "yiyang5055",
-    "author_url": "https://github.com/yiyang5055",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86167",
-    "pr_number": 86167,
-    "kinds": ["cleanup"],
-    "sigs": ["node"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86210": {
-    "commit": "3ccdad175a80a73c8a756e2f35b7ac1d6a98d106",
-    "text": "The following features are unconditionally enabled and the corresponding `--feature-gates` flags have been removed: `PodPriority`, `TaintNodesByCondition`, `ResourceQuotaScopeSelectors` and `ScheduleDaemonSetPods`",
-    "markdown": "The following features are unconditionally enabled and the corresponding `--feature-gates` flags have been removed: `PodPriority`, `TaintNodesByCondition`, `ResourceQuotaScopeSelectors` and `ScheduleDaemonSetPods` ([#86210](https://github.com/kubernetes/kubernetes/pull/86210), [@draveness](https://github.com/draveness))",
-    "author": "draveness",
-    "author_url": "https://github.com/draveness",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86210",
-    "pr_number": 86210,
-    "kinds": ["cleanup"],
-    "sigs": ["apps", "scheduling"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86251": {
-    "commit": "49bc6966149a92e02aaee0245881c6a4a83dc007",
-    "text": "New metric kubelet_pleg_last_seen_seconds to aid diagnosis of PLEG not healthy issues.",
-    "markdown": "New metric kubelet_pleg_last_seen_seconds to aid diagnosis of PLEG not healthy issues. ([#86251](https://github.com/kubernetes/kubernetes/pull/86251), [@bboreham](https://github.com/bboreham))\n\n  Courtesy of SIG Node",
-    "author": "bboreham",
-    "author_url": "https://github.com/bboreham",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86251",
-    "pr_number": 86251,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86282": {
-    "commit": "be26fbc638ff5ed952ccae70e68baa712b3c60c8",
-    "text": "kubelet: resource metrics endpoint `/metrics/resource/v1alpha1` as well as all metrics under this endpoint have been deprecated.\nPlease convert to the following metrics emitted by endpoint `/metrics/resource`:\n- scrape_error --\u003e scrape_error\n- node_cpu_usage_seconds_total --\u003e node_cpu_usage_seconds\n- node_memory_working_set_bytes --\u003e node_memory_working_set_bytes\n- container_cpu_usage_seconds_total --\u003e container_cpu_usage_seconds\n- container_memory_working_set_bytes --\u003e container_memory_working_set_bytes\n- scrape_error --\u003e scrape_error",
-    "markdown": "kubelet: resource metrics endpoint `/metrics/resource/v1alpha1` as well as all metrics under this endpoint have been deprecated.\n  Please convert to the following metrics emitted by endpoint `/metrics/resource`:\n  - scrape_error --\u003e scrape_error\n  - node_cpu_usage_seconds_total --\u003e node_cpu_usage_seconds\n  - node_memory_working_set_bytes --\u003e node_memory_working_set_bytes\n  - container_cpu_usage_seconds_total --\u003e container_cpu_usage_seconds\n  - container_memory_working_set_bytes --\u003e container_memory_working_set_bytes\n  - scrape_error --\u003e scrape_error ([#86282](https://github.com/kubernetes/kubernetes/pull/86282), [@RainbowMango](https://github.com/RainbowMango))\n\n  Courtesy of SIG Node",
-    "author": "RainbowMango",
-    "author_url": "https://github.com/RainbowMango",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86282",
-    "pr_number": 86282,
-    "areas": ["kubelet"],
-    "kinds": ["feature"],
-    "sigs": ["node"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86338": {
-    "commit": "6f4ec3e0d6e1c1e9947091aedcaee882753363d6",
-    "text": "option `preConfiguredBackendPoolLoadBalancerTypes` is added to azure cloud provider for the pre-configured load balancers, possible values: `\"\"`, `\"internal\"`, \"external\"`, `\"all\"`",
-    "markdown": "option `preConfiguredBackendPoolLoadBalancerTypes` is added to azure cloud provider for the pre-configured load balancers, possible values: `\"\"`, `\"internal\"`, \"external\"`, `\"all\"` ([#86338](https://github.com/kubernetes/kubernetes/pull/86338), [@gossion](https://github.com/gossion))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "gossion",
-    "author_url": "https://github.com/gossion",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86338",
-    "pr_number": 86338,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86351": {
-    "commit": "ac4079a5cc22ced66c868f6fd26f8e2497765e51",
-    "text": "fix: azure data disk should use same key as os disk by default",
-    "markdown": "fix: azure data disk should use same key as os disk by default ([#86351](https://github.com/kubernetes/kubernetes/pull/86351), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86351",
-    "pr_number": 86351,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86369": {
-    "commit": "84198bf1206c92a14a5f237c4161d5138886492e",
-    "text": "`AlwaysCheckAllPredicates` is deprecated in scheduler Policy API.",
-    "markdown": "`AlwaysCheckAllPredicates` is deprecated in scheduler Policy API. ([#86369](https://github.com/kubernetes/kubernetes/pull/86369), [@Huang-Wei](https://github.com/Huang-Wei))",
-    "author": "Huang-Wei",
-    "author_url": "https://github.com/Huang-Wei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86369",
-    "pr_number": 86369,
-    "kinds": ["api-change", "cleanup", "deprecation"],
-    "sigs": ["scheduling"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86377": {
-    "commit": "37d9c22abe5921da1d9c61bc92d054eebfb01515",
-    "text": "Introduce Alpha field `Immutable` in both Secret and ConfigMap objects to mark their contents as immutable. The implementation is hidden behind feature gate `ImmutableEphemeralVolumes` (currently in Alpha stage).",
-    "markdown": "Introduce Alpha field `Immutable` in both Secret and ConfigMap objects to mark their contents as immutable. The implementation is hidden behind feature gate `ImmutableEphemeralVolumes` (currently in Alpha stage). ([#86377](https://github.com/kubernetes/kubernetes/pull/86377), [@wojtek-t](https://github.com/wojtek-t))",
-    "author": "wojtek-t",
-    "author_url": "https://github.com/wojtek-t",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86377",
-    "pr_number": 86377,
-    "areas": ["kubectl", "test"],
-    "kinds": ["api-change"],
-    "sigs": ["apps", "cli", "testing"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86412": {
-    "commit": "3577447764103cc3204b66307d651ac2be0ec83f",
-    "text": "Fixes issue where AAD token obtained by kubectl is incompatible with on-behalf-of flow and oidc.\nThe audience claim before this fix has \"spn:\" prefix. After this fix, \"spn:\" prefix is omitted.",
-    "markdown": "Fixes issue where AAD token obtained by kubectl is incompatible with on-behalf-of flow and oidc.\n  The audience claim before this fix has \"spn:\" prefix. After this fix, \"spn:\" prefix is omitted. ([#86412](https://github.com/kubernetes/kubernetes/pull/86412), [@weinong](https://github.com/weinong))",
-    "author": "weinong",
-    "author_url": "https://github.com/weinong",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86412",
-    "pr_number": 86412,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery", "auth", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86425": {
-    "commit": "9f841bf42396daf1ed4b7ab47c1e1708bfb028b3",
-    "text": "hollow-node do not use remote CRI anymore",
-    "markdown": "hollow-node do not use remote CRI anymore ([#86425](https://github.com/kubernetes/kubernetes/pull/86425), [@jkaniuk](https://github.com/jkaniuk))",
-    "author": "jkaniuk",
-    "author_url": "https://github.com/jkaniuk",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86425",
-    "pr_number": 86425,
-    "areas": ["kubelet"],
-    "kinds": ["bug"],
-    "sigs": ["node", "scalability"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86493": {
-    "commit": "3760d24674d9eac92284c5d19c4a774aceab1b7f",
-    "text": "kubeadm: use bind-address option to configure the kube-controller-manager and kube-scheduler http probes",
-    "markdown": "kubeadm: use bind-address option to configure the kube-controller-manager and kube-scheduler http probes ([#86493](https://github.com/kubernetes/kubernetes/pull/86493), [@aojea](https://github.com/aojea))",
-    "author": "aojea",
-    "author_url": "https://github.com/aojea",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86493",
-    "pr_number": 86493,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86496": {
-    "commit": "06fb3eb582187931a22c9fc57024063d91d965f8",
-    "text": "Adds back support for AlwaysCheckAllPredicates flag.",
-    "markdown": "Adds back support for AlwaysCheckAllPredicates flag. ([#86496](https://github.com/kubernetes/kubernetes/pull/86496), [@ahg-g](https://github.com/ahg-g))",
-    "author": "ahg-g",
-    "author_url": "https://github.com/ahg-g",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86496",
-    "pr_number": 86496,
-    "kinds": ["bug"],
-    "sigs": ["scheduling"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86502": {
-    "commit": "4f70231ce7736cc748f76526c98955f86c667a41",
-    "text": "Fix issue \u0026#35;85805 about resource not found in azure cloud provider when lb specified in other resource group.",
-    "markdown": "Fix issue \u0026#35;85805 about resource not found in azure cloud provider when lb specified in other resource group. ([#86502](https://github.com/kubernetes/kubernetes/pull/86502), [@levimm](https://github.com/levimm))",
-    "author": "levimm",
-    "author_url": "https://github.com/levimm",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86502",
-    "pr_number": 86502,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86515": {
-    "commit": "92e0de32698864f4dce876d79e80386261e3e18c",
-    "text": "Azure global rate limit is switched to per-client. A set of new rate limit configure options are introduced, including routeRateLimit, SubnetsRateLimit, InterfaceRateLimit, RouteTableRateLimit, LoadBalancerRateLimit, PublicIPAddressRateLimit, SecurityGroupRateLimit, VirtualMachineRateLimit, StorageAccountRateLimit, DiskRateLimit, SnapshotRateLimit, VirtualMachineScaleSetRateLimit and VirtualMachineSizeRateLimit.\n\nThe original rate limit options would be default values for those new client's rate limiter.",
-    "markdown": "Azure global rate limit is switched to per-client. A set of new rate limit configure options are introduced, including routeRateLimit, SubnetsRateLimit, InterfaceRateLimit, RouteTableRateLimit, LoadBalancerRateLimit, PublicIPAddressRateLimit, SecurityGroupRateLimit, VirtualMachineRateLimit, StorageAccountRateLimit, DiskRateLimit, SnapshotRateLimit, VirtualMachineScaleSetRateLimit and VirtualMachineSizeRateLimit.\n  \n  The original rate limit options would be default values for those new client's rate limiter. ([#86515](https://github.com/kubernetes/kubernetes/pull/86515), [@feiskyer](https://github.com/feiskyer))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86515",
-    "pr_number": 86515,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86534": {
-    "commit": "5e31799701123c50025567b8534e1a62dbc0e9f6",
-    "text": "Fixes v1.17.0 regression in --service-cluster-ip-range handling with IPv4 ranges larger than 65536 IP addresses",
-    "markdown": "Fixes v1.17.0 regression in --service-cluster-ip-range handling with IPv4 ranges larger than 65536 IP addresses ([#86534](https://github.com/kubernetes/kubernetes/pull/86534), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86534",
-    "pr_number": 86534,
-    "kinds": ["bug"],
-    "sigs": ["network"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86584": {
-    "commit": "180e1c9f5159a22576ee866a82a7dc8c26cc1c41",
-    "text": "Marked scheduler's metrics scheduling_algorithm_predicate_evaluation_seconds and\nscheduling_algorithm_priority_evaluation_seconds as deprecated. Those are replaced by framework_extension_point_duration_seconds[extenstion_point=\"Filter\"] and framework_extension_point_duration_seconds[extenstion_point=\"Score\"] respectively.",
-    "markdown": "Marked scheduler's metrics scheduling_algorithm_predicate_evaluation_seconds and\n  scheduling_algorithm_priority_evaluation_seconds as deprecated. Those are replaced by framework_extension_point_duration_seconds[extenstion_point=\"Filter\"] and framework_extension_point_duration_seconds[extenstion_point=\"Score\"] respectively. ([#86584](https://github.com/kubernetes/kubernetes/pull/86584), [@xiaoanyunfei](https://github.com/xiaoanyunfei))",
-    "author": "xiaoanyunfei",
-    "author_url": "https://github.com/xiaoanyunfei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86584",
-    "pr_number": 86584,
-    "kinds": ["cleanup", "deprecation"],
-    "sigs": ["scheduling"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86586": {
-    "commit": "fdfcb00edee127df1f9d27f0b858271ed8959421",
-    "text": "Marked scheduler's scheduling_duration_seconds Summary metric as deprecated",
-    "markdown": "Marked scheduler's scheduling_duration_seconds Summary metric as deprecated ([#86586](https://github.com/kubernetes/kubernetes/pull/86586), [@xiaoanyunfei](https://github.com/xiaoanyunfei))",
-    "author": "xiaoanyunfei",
-    "author_url": "https://github.com/xiaoanyunfei",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86586",
-    "pr_number": 86586,
-    "kinds": ["cleanup", "deprecation"],
-    "sigs": ["scheduling"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86609": {
-    "commit": "f772c9017c16bbc983e59a19162814629a467cef",
-    "text": "kubeadm: add support for the \"ci/k8s-master\" version label as a replacement for \"ci-cross/*\", which no longer exists.",
-    "markdown": "kubeadm: add support for the \"ci/k8s-master\" version label as a replacement for \"ci-cross/*\", which no longer exists. ([#86609](https://github.com/kubernetes/kubernetes/pull/86609), [@Pensu](https://github.com/Pensu))",
-    "author": "Pensu",
-    "author_url": "https://github.com/Pensu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86609",
-    "pr_number": 86609,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86612": {
-    "commit": "40320be679e9bc85632a621dc2fa12af39f37c6d",
-    "text": "fix: azure disk could not mounted on Standard_DC4s/DC2s instances",
-    "markdown": "fix: azure disk could not mounted on Standard_DC4s/DC2s instances ([#86612](https://github.com/kubernetes/kubernetes/pull/86612), [@andyzhangx](https://github.com/andyzhangx))",
-    "author": "andyzhangx",
-    "author_url": "https://github.com/andyzhangx",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86612",
-    "pr_number": 86612,
-    "areas": ["provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider", "storage"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86646": {
-    "commit": "19ecd690fa1ff0747db6388714c0569dfc2548bd",
-    "text": "For subprotocol negotiation, both client and server protocol is required now.",
-    "markdown": "For subprotocol negotiation, both client and server protocol is required now. ([#86646](https://github.com/kubernetes/kubernetes/pull/86646), [@tedyu](https://github.com/tedyu))",
-    "author": "tedyu",
-    "author_url": "https://github.com/tedyu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86646",
-    "pr_number": 86646,
-    "areas": ["kubelet"],
-    "kinds": ["cleanup"],
-    "sigs": ["api-machinery", "node"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86693": {
-    "commit": "c72d5fd2556f90669fc3321061dba658cad077cf",
-    "text": "If a required flag is not provided to a command, the user will only see the required flag error message, instead of the entire usage menu.",
-    "markdown": "If a required flag is not provided to a command, the user will only see the required flag error message, instead of the entire usage menu. ([#86693](https://github.com/kubernetes/kubernetes/pull/86693), [@sallyom](https://github.com/sallyom))",
-    "author": "sallyom",
-    "author_url": "https://github.com/sallyom",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86693",
-    "pr_number": 86693,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86705": {
-    "commit": "09cb73a554d3218f0267ba28de8a5fba47fd6335",
-    "text": "kubeadm: tolerate whitespace when validating certificate authority PEM data in kubeconfig files",
-    "markdown": "kubeadm: tolerate whitespace when validating certificate authority PEM data in kubeconfig files ([#86705](https://github.com/kubernetes/kubernetes/pull/86705), [@neolit123](https://github.com/neolit123))",
-    "author": "neolit123",
-    "author_url": "https://github.com/neolit123",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86705",
-    "pr_number": 86705,
-    "areas": ["kubeadm"],
-    "kinds": ["bug"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86740": {
-    "commit": "b046dbce1332f6d67e8c3adf03a7b62f605b759c",
-    "text": "Azure VMSS/VMSSVM clients now suppress requests on throttling",
-    "markdown": "Azure VMSS/VMSSVM clients now suppress requests on throttling ([#86740](https://github.com/kubernetes/kubernetes/pull/86740), [@feiskyer](https://github.com/feiskyer))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86740",
-    "pr_number": 86740,
-    "areas": ["cloudprovider", "dependency", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86801": {
-    "commit": "073da1588a9ec79d673f3356734e22756f06c05e",
-    "text": "fix a bug that orphan revision cannot be adopted and statefulset cannot be synced",
-    "markdown": "fix a bug that orphan revision cannot be adopted and statefulset cannot be synced ([#86801](https://github.com/kubernetes/kubernetes/pull/86801), [@likakuli](https://github.com/likakuli))",
-    "author": "likakuli",
-    "author_url": "https://github.com/likakuli",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86801",
-    "pr_number": 86801,
-    "kinds": ["bug"],
-    "sigs": ["apps"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86861": {
-    "commit": "916edd922e528f2f27542d3ca5967ca11fe153f8",
-    "text": "kubeadm upgrades always persist the etcd backup for stacked",
-    "markdown": "kubeadm upgrades always persist the etcd backup for stacked ([#86861](https://github.com/kubernetes/kubernetes/pull/86861), [@SataQiu](https://github.com/SataQiu))",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86861",
-    "pr_number": 86861,
-    "areas": ["kubeadm"],
-    "kinds": ["cleanup"],
-    "sigs": ["cluster-lifecycle"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86899": {
-    "commit": "3e8155ead3d890697141d4946ec5fdd5b0dab192",
-    "text": "kubeadm: support automatic retry after failing to pull image",
-    "markdown": "kubeadm: support automatic retry after failing to pull image ([#86899](https://github.com/kubernetes/kubernetes/pull/86899), [@SataQiu](https://github.com/SataQiu))\n\n  Courtesy of SIG Cluster Lifecycle",
-    "author": "SataQiu",
-    "author_url": "https://github.com/SataQiu",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86899",
-    "pr_number": 86899,
-    "areas": ["kubeadm"],
-    "kinds": ["feature"],
-    "sigs": ["cluster-lifecycle"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86940": {
-    "commit": "edad4bbfc824215fc254096dfbbd1b2ab8ce6781",
-    "text": "Add VolumeBinder method to FrameworkHandle interface, which allows user to get the volume binder when implementing scheduler framework plugins.",
-    "markdown": "Add VolumeBinder method to FrameworkHandle interface, which allows user to get the volume binder when implementing scheduler framework plugins. ([#86940](https://github.com/kubernetes/kubernetes/pull/86940), [@skilxn-go](https://github.com/skilxn-go))\n\n  Courtesy of SIG Scheduling",
-    "author": "skilxn-go",
-    "author_url": "https://github.com/skilxn-go",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86940",
-    "pr_number": 86940,
-    "kinds": ["feature"],
-    "sigs": ["scheduling"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "86952": {
-    "commit": "6026a6a1515244855d10a51857af2e8c2fd26f37",
-    "text": "fix kubectl annotate error when local=true is set",
-    "markdown": "fix kubectl annotate error when local=true is set ([#86952](https://github.com/kubernetes/kubernetes/pull/86952), [@zhouya0](https://github.com/zhouya0))",
-    "author": "zhouya0",
-    "author_url": "https://github.com/zhouya0",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/86952",
-    "pr_number": 86952,
-    "areas": ["kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": ["cli"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "87029": {
-    "commit": "af46f2cdd2a72ef808f99715c978d58a4603871b",
-    "text": "Fixes service account token admission error in clusters that do not run the service account token controller",
-    "markdown": "Fixes service account token admission error in clusters that do not run the service account token controller ([#87029](https://github.com/kubernetes/kubernetes/pull/87029), [@liggitt](https://github.com/liggitt))",
-    "author": "liggitt",
-    "author_url": "https://github.com/liggitt",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87029",
-    "pr_number": 87029,
-    "kinds": ["bug"],
-    "sigs": ["auth"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "87035": {
-    "commit": "d2a526bc0eb0961966584749935e7a28bd769adc",
-    "text": "provider/azure: Network security groups can now be in a separate resource group.",
-    "markdown": "provider/azure: Network security groups can now be in a separate resource group. ([#87035](https://github.com/kubernetes/kubernetes/pull/87035), [@CecileRobertMichon](https://github.com/CecileRobertMichon))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "CecileRobertMichon",
-    "author_url": "https://github.com/CecileRobertMichon",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87035",
-    "pr_number": 87035,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "87043": {
-    "commit": "240782c7f7ee1a810207b532f970ad0fe5812939",
-    "text": "Fixed a bug which could prevent a provider ID from ever being set for node if an error occurred determining the provider ID when the node was added.",
-    "markdown": "Fixed a bug which could prevent a provider ID from ever being set for node if an error occurred determining the provider ID when the node was added. ([#87043](https://github.com/kubernetes/kubernetes/pull/87043), [@zjs](https://github.com/zjs))",
-    "author": "zjs",
-    "author_url": "https://github.com/zjs",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87043",
-    "pr_number": 87043,
-    "areas": ["cloudprovider"],
-    "kinds": ["bug"],
-    "sigs": ["apps", "cloud-provider"],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "87044": {
-    "commit": "36e40fb850293076b415ae3d376f5f81dc897105",
-    "text": "TODO",
-    "markdown": "TODO ([#87044](https://github.com/kubernetes/kubernetes/pull/87044), [@jennybuckley](https://github.com/jennybuckley))\n\n  Courtesy of SIG API Machinery",
-    "author": "jennybuckley",
-    "author_url": "https://github.com/jennybuckley",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87044",
-    "pr_number": 87044,
-    "areas": ["apiserver"],
-    "kinds": ["feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "87122": {
-    "commit": "93595eb4eb933bfed90b1f4a1f5597caccf7ccfd",
-    "text": "Azure network and VM clients now suppress requests on throttling",
-    "markdown": "Azure network and VM clients now suppress requests on throttling ([#87122](https://github.com/kubernetes/kubernetes/pull/87122), [@feiskyer](https://github.com/feiskyer))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87122",
-    "pr_number": 87122,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "87203": {
-    "commit": "0c8dcc59f099f69240ffd83eb2eae1cb6e1c68aa",
-    "text": "All apiservers log request lines in a more greppable format.",
-    "markdown": "All apiservers log request lines in a more greppable format. ([#87203](https://github.com/kubernetes/kubernetes/pull/87203), [@lavalamp](https://github.com/lavalamp))",
-    "author": "lavalamp",
-    "author_url": "https://github.com/lavalamp",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87203",
-    "pr_number": 87203,
-    "areas": ["apiserver"],
-    "kinds": ["bug"],
-    "sigs": ["api-machinery"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "87213": {
-    "commit": "a25c6aa35067187800ef2a0928646f5d23268764",
-    "text": "CustomResourceDefinition status fields are no longer required for client validation when submitting manifests.",
-    "markdown": "CustomResourceDefinition status fields are no longer required for client validation when submitting manifests. ([#87213](https://github.com/kubernetes/kubernetes/pull/87213), [@hasheddan](https://github.com/hasheddan))\n\n  Courtesy of SIG API Machinery",
-    "author": "hasheddan",
-    "author_url": "https://github.com/hasheddan",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87213",
-    "pr_number": 87213,
-    "areas": ["custom-resources"],
-    "kinds": ["api-change", "feature"],
-    "sigs": ["api-machinery"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "87246": {
-    "commit": "6413f1ee2be99fe5ba18844a2646fb15d462bec4",
-    "text": "Fix the bug PIP's DNS is deleted if no DNS label service annotation isn't set.",
-    "markdown": "Fix the bug PIP's DNS is deleted if no DNS label service annotation isn't set. ([#87246](https://github.com/kubernetes/kubernetes/pull/87246), [@nilo19](https://github.com/nilo19))",
-    "author": "nilo19",
-    "author_url": "https://github.com/nilo19",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87246",
-    "pr_number": 87246,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["bug"],
-    "sigs": ["cloud-provider"],
-    "release_version": "1.18.0-alpha.2"
-  },
-  "87306": {
-    "commit": "32e72a54fc24bebe8d0bb311d734731f11131206",
-    "text": "Azure storage clients now suppress requests on throttling",
-    "markdown": "Azure storage clients now suppress requests on throttling ([#87306](https://github.com/kubernetes/kubernetes/pull/87306), [@feiskyer](https://github.com/feiskyer))\n\n  Courtesy of SIG Cloud Provider",
-    "author": "feiskyer",
-    "author_url": "https://github.com/feiskyer",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87306",
-    "pr_number": 87306,
-    "areas": ["cloudprovider", "provider/azure"],
-    "kinds": ["feature"],
-    "sigs": ["cloud-provider"],
-    "feature": true,
-    "release_version": "1.18.0-alpha.2"
-  },
-  "87326": {
-    "commit": "33aa665c344f91c10a15f0efda730e171b544d96",
-    "text": "Bump golang/mock version to v1.3.1",
-    "markdown": "Bump golang/mock version to v1.3.1 ([#87326](https://github.com/kubernetes/kubernetes/pull/87326), [@wawa0210](https://github.com/wawa0210))",
-    "author": "wawa0210",
-    "author_url": "https://github.com/wawa0210",
-    "pr_url": "https://github.com/kubernetes/kubernetes/pull/87326",
-    "pr_number": 87326,
-    "areas": ["apiserver", "cloudprovider", "dependency", "kubectl"],
-    "kinds": ["cleanup"],
-    "sigs": [
-      "api-machinery",
-      "cli",
-      "cloud-provider",
-      "cluster-lifecycle",
-      "instrumentation",
-      "node"
-    ],
-    "duplicate": true,
-    "release_version": "1.18.0-alpha.2"
-  }
-}
diff --git a/src/environments/assets.ts b/src/environments/assets.ts
index 872ba83f..ad292b32 100644
--- a/src/environments/assets.ts
+++ b/src/environments/assets.ts
@@ -1,35 +1 @@
-export const assets = [
-  'assets/release-notes-1.18.0-alpha.2.json',
-  'assets/release-notes-1.18.0-alpha.1.json',
-  'assets/release-notes-1.17.2.json',
-  'assets/release-notes-1.17.1.json',
-  'assets/release-notes-1.17.0.json',
-  'assets/release-notes-1.16.6.json',
-  'assets/release-notes-1.16.5.json',
-  'assets/release-notes-1.16.4.json',
-  'assets/release-notes-1.16.3.json',
-  'assets/release-notes-1.16.2.json',
-  'assets/release-notes-1.16.1.json',
-  'assets/release-notes-1.16.0.json',
-  'assets/release-notes-1.15.9.json',
-  'assets/release-notes-1.15.8.json',
-  'assets/release-notes-1.15.7.json',
-  'assets/release-notes-1.15.6.json',
-  'assets/release-notes-1.15.5.json',
-  'assets/release-notes-1.15.4.json',
-  'assets/release-notes-1.15.3.json',
-  'assets/release-notes-1.15.2.json',
-  'assets/release-notes-1.15.1.json',
-  'assets/release-notes-1.15.0.json',
-  'assets/release-notes-1.14.10.json',
-  'assets/release-notes-1.14.9.json',
-  'assets/release-notes-1.14.8.json',
-  'assets/release-notes-1.14.7.json',
-  'assets/release-notes-1.14.6.json',
-  'assets/release-notes-1.14.5.json',
-  'assets/release-notes-1.14.4.json',
-  'assets/release-notes-1.14.3.json',
-  'assets/release-notes-1.14.2.json',
-  'assets/release-notes-1.14.1.json',
-  'assets/release-notes-1.14.0.json',
-];
+export const assets = ['assets/keps.json'];
diff --git a/src/index.html b/src/index.html
index 9af17613..390c20b7 100644
--- a/src/index.html
+++ b/src/index.html
@@ -2,7 +2,7 @@
 <html lang="en">
   <head>
     <meta charset="utf-8" />
-    <title>Kubernetes Release Notes</title>
+    <title>Kubernetes Enhancement Proposals</title>
     <base href="/" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <link rel="shortcut icon" type=image/png href=favicon.png>