typhoon/addons/grafana/dashboards-k8s.yaml

apiVersion: v1
data:
  apiserver.json: |-
    {
      "__inputs": [

      ],
      "__requires": [

      ],
      "annotations": {
        "list": [

        ]
      },
      "editable": false,
      "gnetId": null,
      "graphTooltip": 0,
      "hideControls": false,
      "id": null,
      "links": [

      ],
      "panels": [
        {
          "content": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.",
          "datasource": null,
          "description": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.",
          "gridPos": {
            "h": 2,
            "w": 24,
            "x": 0,
            "y": 0
          },
          "id": 2,
          "mode": "markdown",
          "span": 12,
          "title": "Notice",
          "type": "text"
        }
      ],
      "refresh": "10s",
      "rows": [
        {
          "collapse": false,
          "collapsed": false,
          "panels": [
            {
              "cacheTimeout": null,
              "colorBackground": false,
              "colorValue": false,
              "colors": [
                "#299c46",
                "rgba(237, 129, 40, 0.89)",
                "#d44a3a"
              ],
              "datasource": "$datasource",
              "decimals": 3,
              "description": "How many percent of requests (both read and write) in 30 days have been answered successfully and fast enough?",
              "format": "percentunit",
              "gauge": {
                "maxValue": 100,
                "minValue": 0,
                "show": false,
                "thresholdLabels": false,
                "thresholdMarkers": true
              },
              "gridPos": {

              },
              "id": 3,
              "interval": null,
              "links": [

              ],
              "mappingType": 1,
              "mappingTypes": [
                {
                  "name": "value to text",
                  "value": 1
                },
                {
                  "name": "range to text",
                  "value": 2
                }
              ],
              "maxDataPoints": 100,
              "nullPointMode": "connected",
              "nullText": null,
              "postfix": "",
              "postfixFontSize": "50%",
              "prefix": "",
              "prefixFontSize": "50%",
              "rangeMaps": [
                {
                  "from": "null",
                  "text": "N/A",
                  "to": "null"
                }
              ],
              "span": 4,
              "sparkline": {
                "fillColor": "rgba(31, 118, 189, 0.18)",
                "full": false,
                "lineColor": "rgb(31, 120, 193)",
                "show": false
              },
              "tableColumn": "",
              "targets": [
                {
                  "expr": "apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "",
                  "refId": "A"
                }
              ],
              "thresholds": "",
              "title": "Availability (30d) > 99.000%",
              "tooltip": {
                "shared": false
              },
              "type": "singlestat",
              "valueFontSize": "80%",
              "valueMaps": [
                {
                  "op": "=",
                  "text": "N/A",
                  "value": "null"
                }
              ],
              "valueName": "avg"
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "decimals": 3,
              "description": "How much error budget is left looking at our 0.990% availability guarantees?",
              "fill": 10,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 4,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 8,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "100 * (apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"} - 0.990000)",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "errorbudget",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "ErrorBudget (30d) > 99.000%",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "decimals": 3,
                  "format": "percentunit",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "decimals": 3,
                  "format": "percentunit",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        },
        {
          "collapse": false,
          "collapsed": false,
          "panels": [
            {
              "cacheTimeout": null,
              "colorBackground": false,
              "colorValue": false,
              "colors": [
                "#299c46",
                "rgba(237, 129, 40, 0.89)",
                "#d44a3a"
              ],
              "datasource": "$datasource",
              "decimals": 3,
              "description": "How many percent of read requests (LIST,GET) in 30 days have been answered successfully and fast enough?",
              "format": "percentunit",
              "gauge": {
                "maxValue": 100,
                "minValue": 0,
                "show": false,
                "thresholdLabels": false,
                "thresholdMarkers": true
              },
              "gridPos": {

              },
              "id": 5,
              "interval": null,
              "links": [

              ],
              "mappingType": 1,
              "mappingTypes": [
                {
                  "name": "value to text",
                  "value": 1
                },
                {
                  "name": "range to text",
                  "value": 2
                }
              ],
              "maxDataPoints": 100,
              "nullPointMode": "connected",
              "nullText": null,
              "postfix": "",
              "postfixFontSize": "50%",
              "prefix": "",
              "prefixFontSize": "50%",
              "rangeMaps": [
                {
                  "from": "null",
                  "text": "N/A",
                  "to": "null"
                }
              ],
              "span": 3,
              "sparkline": {
                "fillColor": "rgba(31, 118, 189, 0.18)",
                "full": false,
                "lineColor": "rgb(31, 120, 193)",
                "show": false
              },
              "tableColumn": "",
              "targets": [
                {
                  "expr": "apiserver_request:availability30d{verb=\"read\", cluster=\"$cluster\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "",
                  "refId": "A"
                }
              ],
              "thresholds": "",
              "title": "Read Availability (30d)",
              "tooltip": {
                "shared": false
              },
              "type": "singlestat",
              "valueFontSize": "80%",
              "valueMaps": [
                {
                  "op": "=",
                  "text": "N/A",
                  "value": "null"
                }
              ],
              "valueName": "avg"
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "How many read requests (LIST,GET) per second do the apiservers get by code?",
              "fill": 10,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 6,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [
                {
                  "alias": "/2../i",
                  "color": "#56A64B"
                },
                {
                  "alias": "/3../i",
                  "color": "#F2CC0C"
                },
                {
                  "alias": "/4../i",
                  "color": "#3274D9"
                },
                {
                  "alias": "/5../i",
                  "color": "#E02F44"
                }
              ],
              "spaceLength": 10,
              "span": 3,
              "stack": true,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{ code }}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Read SLI - Requests",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "reqps",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "reqps",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "How many percent of read requests (LIST,GET) per second are returned with errors (5xx)?",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 7,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 3,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{ resource }}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Read SLI - Errors",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "percentunit",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "percentunit",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "How many seconds is the 99th percentile for reading (LIST|GET) a given resource?",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 8,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 3,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{verb=\"read\", cluster=\"$cluster\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{ resource }}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Read SLI - Duration",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        },
        {
          "collapse": false,
          "collapsed": false,
          "panels": [
            {
              "cacheTimeout": null,
              "colorBackground": false,
              "colorValue": false,
              "colors": [
                "#299c46",
                "rgba(237, 129, 40, 0.89)",
                "#d44a3a"
              ],
              "datasource": "$datasource",
              "decimals": 3,
              "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) in 30 days have been answered successfully and fast enough?",
              "format": "percentunit",
              "gauge": {
                "maxValue": 100,
                "minValue": 0,
                "show": false,
                "thresholdLabels": false,
                "thresholdMarkers": true
              },
              "gridPos": {

              },
              "id": 9,
              "interval": null,
              "links": [

              ],
              "mappingType": 1,
              "mappingTypes": [
                {
                  "name": "value to text",
                  "value": 1
                },
                {
                  "name": "range to text",
                  "value": 2
                }
              ],
              "maxDataPoints": 100,
              "nullPointMode": "connected",
              "nullText": null,
              "postfix": "",
              "postfixFontSize": "50%",
              "prefix": "",
              "prefixFontSize": "50%",
              "rangeMaps": [
                {
                  "from": "null",
                  "text": "N/A",
                  "to": "null"
                }
              ],
              "span": 3,
              "sparkline": {
                "fillColor": "rgba(31, 118, 189, 0.18)",
                "full": false,
                "lineColor": "rgb(31, 120, 193)",
                "show": false
              },
              "tableColumn": "",
              "targets": [
                {
                  "expr": "apiserver_request:availability30d{verb=\"write\", cluster=\"$cluster\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "",
                  "refId": "A"
                }
              ],
              "thresholds": "",
              "title": "Write Availability (30d)",
              "tooltip": {
                "shared": false
              },
              "type": "singlestat",
              "valueFontSize": "80%",
              "valueMaps": [
                {
                  "op": "=",
                  "text": "N/A",
                  "value": "null"
                }
              ],
              "valueName": "avg"
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "How many write requests (POST|PUT|PATCH|DELETE) per second do the apiservers get by code?",
              "fill": 10,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 10,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [
                {
                  "alias": "/2../i",
                  "color": "#56A64B"
                },
                {
                  "alias": "/3../i",
                  "color": "#F2CC0C"
                },
                {
                  "alias": "/4../i",
                  "color": "#3274D9"
                },
                {
                  "alias": "/5../i",
                  "color": "#E02F44"
                }
              ],
              "spaceLength": 10,
              "span": 3,
              "stack": true,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{ code }}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Write SLI - Requests",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "reqps",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "reqps",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) per second are returned with errors (5xx)?",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 11,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 3,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{ resource }}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Write SLI - Errors",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "percentunit",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "percentunit",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "How many seconds is the 99th percentile for writing (POST|PUT|PATCH|DELETE) a given resource?",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 12,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 3,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{verb=\"write\", cluster=\"$cluster\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{ resource }}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Write SLI - Duration",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        },
        {
          "collapse": false,
          "collapsed": false,
          "panels": [
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 13,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": false,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 6,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}} {{name}}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Work Queue Add Rate",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "ops",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "ops",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 14,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": false,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 6,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}} {{name}}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Work Queue Depth",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 15,
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": true,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": true
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 12,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name, le))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}} {{name}}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Work Queue Latency",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        },
        {
          "collapse": false,
          "collapsed": false,
          "panels": [
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 16,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Memory",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "bytes",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "bytes",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 17,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[5m])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "CPU usage",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 18,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "go_goroutines{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Goroutines",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        }
      ],
      "schemaVersion": 14,
      "style": "dark",
      "tags": [
        "kubernetes-mixin"
      ],
      "templating": {
        "list": [
          {
            "current": {
              "text": "default",
              "value": "default"
            },
            "hide": 0,
            "label": null,
            "name": "datasource",
            "options": [

            ],
            "query": "prometheus",
            "refresh": 1,
            "regex": "",
            "type": "datasource"
          },
          {
            "allValue": null,
            "current": {

            },
            "datasource": "$datasource",
            "hide": 2,
            "includeAll": false,
            "label": "cluster",
            "multi": false,
            "name": "cluster",
            "options": [

            ],
            "query": "label_values(apiserver_request_total, cluster)",
            "refresh": 2,
            "regex": "",
            "sort": 1,
            "tagValuesQuery": "",
            "tags": [

            ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
          },
          {
            "allValue": null,
            "current": {

            },
            "datasource": "$datasource",
            "hide": 0,
            "includeAll": true,
            "label": null,
            "multi": false,
            "name": "instance",
            "options": [

            ],
            "query": "label_values(apiserver_request_total{job=\"apiserver\", cluster=\"$cluster\"}, instance)",
            "refresh": 2,
            "regex": "",
            "sort": 1,
            "tagValuesQuery": "",
            "tags": [

            ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
          }
        ]
      },
      "time": {
        "from": "now-1h",
        "to": "now"
      },
      "timepicker": {
        "refresh_intervals": [
          "5s",
          "10s",
          "30s",
          "1m",
          "5m",
          "15m",
          "30m",
          "1h",
          "2h",
          "1d"
        ],
        "time_options": [
          "5m",
          "15m",
          "1h",
          "6h",
          "12h",
          "24h",
          "2d",
          "7d",
          "30d"
        ]
      },
      "timezone": "UTC",
      "title": "Kubernetes / API server",
      "uid": "09ec8aa1e996d6ffcd6817bbaff4db1b",
      "version": 0
    }
  controller-manager.json: |-
    {
      "__inputs": [

      ],
      "__requires": [

      ],
      "annotations": {
        "list": [

        ]
      },
      "editable": false,
      "gnetId": null,
      "graphTooltip": 0,
      "hideControls": false,
      "id": null,
      "links": [

      ],
      "refresh": "10s",
      "rows": [
        {
          "collapse": false,
          "collapsed": false,
          "panels": [
            {
              "cacheTimeout": null,
              "colorBackground": false,
              "colorValue": false,
              "colors": [
                "#299c46",
                "rgba(237, 129, 40, 0.89)",
                "#d44a3a"
              ],
              "datasource": "$datasource",
              "format": "none",
              "gauge": {
                "maxValue": 100,
                "minValue": 0,
                "show": false,
                "thresholdLabels": false,
                "thresholdMarkers": true
              },
              "gridPos": {

              },
              "id": 2,
              "interval": null,
              "links": [

              ],
              "mappingType": 1,
              "mappingTypes": [
                {
                  "name": "value to text",
                  "value": 1
                },
                {
                  "name": "range to text",
                  "value": 2
                }
              ],
              "maxDataPoints": 100,
              "nullPointMode": "connected",
              "nullText": null,
              "postfix": "",
              "postfixFontSize": "50%",
              "prefix": "",
              "prefixFontSize": "50%",
              "rangeMaps": [
                {
                  "from": "null",
                  "text": "N/A",
                  "to": "null"
                }
              ],
              "span": 2,
              "sparkline": {
                "fillColor": "rgba(31, 118, 189, 0.18)",
                "full": false,
                "lineColor": "rgb(31, 120, 193)",
                "show": false
              },
              "tableColumn": "",
              "targets": [
                {
                  "expr": "sum(up{job=\"kube-controller-manager\"})",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "",
                  "refId": "A"
                }
              ],
              "thresholds": "",
              "title": "Up",
              "tooltip": {
                "shared": false
              },
              "type": "singlestat",
              "valueFontSize": "80%",
              "valueMaps": [
                {
                  "op": "=",
                  "text": "N/A",
                  "value": "null"
                }
              ],
              "valueName": "min"
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 3,
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": true,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": true
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(workqueue_adds_total{job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name)",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}} {{name}}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Work Queue Add Rate",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "ops",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "ops",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        },
        {
          "collapse": false,
          "collapsed": false,
          "panels": [
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 4,
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": true,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": true
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 12,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(workqueue_depth{job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name)",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}} {{name}}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Work Queue Depth",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        },
        {
          "collapse": false,
          "collapsed": false,
          "panels": [
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 5,
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": true,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": true
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 12,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name, le))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}} {{name}}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Work Queue Latency",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        },
        {
          "collapse": false,
          "collapsed": false,
          "panels": [
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 6,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"2..\"}[5m]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "2xx",
                  "refId": "A"
                },
                {
                  "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"3..\"}[5m]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "3xx",
                  "refId": "B"
                },
                {
                  "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"4..\"}[5m]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "4xx",
                  "refId": "C"
                },
                {
                  "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"5..\"}[5m]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "5xx",
                  "refId": "D"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Kube API Request Rate",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "ops",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "ops",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 7,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 8,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{verb}} {{url}}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Post Request Latency 99th Quantile",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        },
        {
          "collapse": false,
          "collapsed": false,
          "panels": [
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 8,
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": true,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": true
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 12,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{verb}} {{url}}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Get Request Latency 99th Quantile",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        },
        {
          "collapse": false,
          "collapsed": false,
          "panels": [
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 9,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "process_resident_memory_bytes{job=\"kube-controller-manager\",instance=~\"$instance\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Memory",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "bytes",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "bytes",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 10,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(process_cpu_seconds_total{job=\"kube-controller-manager\",instance=~\"$instance\"}[5m])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "CPU usage",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 11,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "go_goroutines{job=\"kube-controller-manager\",instance=~\"$instance\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Goroutines",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        }
      ],
      "schemaVersion": 14,
      "style": "dark",
      "tags": [
        "kubernetes-mixin"
      ],
      "templating": {
        "list": [
          {
            "current": {
              "text": "default",
              "value": "default"
            },
            "hide": 0,
            "label": null,
            "name": "datasource",
            "options": [

            ],
            "query": "prometheus",
            "refresh": 1,
            "regex": "",
            "type": "datasource"
          },
          {
            "allValue": null,
            "current": {

            },
            "datasource": "$datasource",
            "hide": 0,
            "includeAll": true,
            "label": null,
            "multi": false,
            "name": "instance",
            "options": [

            ],
            "query": "label_values(process_cpu_seconds_total{job=\"kube-controller-manager\"}, instance)",
            "refresh": 2,
            "regex": "",
            "sort": 1,
            "tagValuesQuery": "",
            "tags": [

            ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
          }
        ]
      },
      "time": {
        "from": "now-1h",
        "to": "now"
      },
      "timepicker": {
        "refresh_intervals": [
          "5s",
          "10s",
          "30s",
          "1m",
          "5m",
          "15m",
          "30m",
          "1h",
          "2h",
          "1d"
        ],
        "time_options": [
          "5m",
          "15m",
          "1h",
          "6h",
          "12h",
          "24h",
          "2d",
          "7d",
          "30d"
        ]
      },
      "timezone": "UTC",
      "title": "Kubernetes / Controller Manager",
      "uid": "72e0e05bef5099e5f049b05fdc429ed4",
      "version": 0
    }
  persistentvolumesusage.json: |-
    {
      "__inputs": [

      ],
      "__requires": [

      ],
      "annotations": {
        "list": [

        ]
      },
      "editable": false,
      "gnetId": null,
      "graphTooltip": 0,
      "hideControls": false,
      "id": null,
      "links": [

      ],
      "refresh": "10s",
      "rows": [
        {
          "collapse": false,
          "collapsed": false,
          "panels": [
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 2,
              "legend": {
                "alignAsTable": true,
                "avg": true,
                "current": true,
                "max": true,
                "min": true,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": true
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 9,
              "stack": true,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "(\n  sum without(instance, node) (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n  -\n  sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "Used Space",
                  "refId": "A"
                },
                {
                  "expr": "sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "Free Space",
                  "refId": "B"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Volume Space Usage",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "bytes",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "bytes",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                }
              ]
            },
            {
              "cacheTimeout": null,
              "colorBackground": false,
              "colorValue": false,
              "colors": [
                "rgba(50, 172, 45, 0.97)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(245, 54, 54, 0.9)"
              ],
              "datasource": "$datasource",
              "format": "percent",
              "gauge": {
                "maxValue": 100,
                "minValue": 0,
                "show": true,
                "thresholdLabels": false,
                "thresholdMarkers": true
              },
              "gridPos": {

              },
              "id": 3,
              "interval": null,
              "links": [

              ],
              "mappingType": 1,
              "mappingTypes": [
                {
                  "name": "value to text",
                  "value": 1
                },
                {
                  "name": "range to text",
                  "value": 2
                }
              ],
              "maxDataPoints": 100,
              "nullPointMode": "connected",
              "nullText": null,
              "postfix": "",
              "postfixFontSize": "50%",
              "prefix": "",
              "prefixFontSize": "50%",
              "rangeMaps": [
                {
                  "from": "null",
                  "text": "N/A",
                  "to": "null"
                }
              ],
              "span": 3,
              "sparkline": {
                "fillColor": "rgba(31, 118, 189, 0.18)",
                "full": false,
                "lineColor": "rgb(31, 120, 193)",
                "show": false
              },
              "tableColumn": "",
              "targets": [
                {
                  "expr": "max without(instance,node) (\n(\n  kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n  -\n  kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n)\n/\nkubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100)\n",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "",
                  "refId": "A"
                }
              ],
              "thresholds": "80, 90",
              "title": "Volume Space Usage",
              "tooltip": {
                "shared": false
              },
              "type": "singlestat",
              "valueFontSize": "80%",
              "valueMaps": [
                {
                  "op": "=",
                  "text": "N/A",
                  "value": "null"
                }
              ],
              "valueName": "current"
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        },
        {
          "collapse": false,
          "collapsed": false,
          "panels": [
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 4,
              "legend": {
                "alignAsTable": true,
                "avg": true,
                "current": true,
                "max": true,
                "min": true,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": true
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 9,
              "stack": true,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "Used inodes",
                  "refId": "A"
                },
                {
                  "expr": "(\n  sum without(instance, node) (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n  -\n  sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": " Free inodes",
                  "refId": "B"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Volume inodes Usage",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "none",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "none",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                }
              ]
            },
            {
              "cacheTimeout": null,
              "colorBackground": false,
              "colorValue": false,
              "colors": [
                "rgba(50, 172, 45, 0.97)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(245, 54, 54, 0.9)"
              ],
              "datasource": "$datasource",
              "format": "percent",
              "gauge": {
                "maxValue": 100,
                "minValue": 0,
                "show": true,
                "thresholdLabels": false,
                "thresholdMarkers": true
              },
              "gridPos": {

              },
              "id": 5,
              "interval": null,
              "links": [

              ],
              "mappingType": 1,
              "mappingTypes": [
                {
                  "name": "value to text",
                  "value": 1
                },
                {
                  "name": "range to text",
                  "value": 2
                }
              ],
              "maxDataPoints": 100,
              "nullPointMode": "connected",
              "nullText": null,
              "postfix": "",
              "postfixFontSize": "50%",
              "prefix": "",
              "prefixFontSize": "50%",
              "rangeMaps": [
                {
                  "from": "null",
                  "text": "N/A",
                  "to": "null"
                }
              ],
              "span": 3,
              "sparkline": {
                "fillColor": "rgba(31, 118, 189, 0.18)",
                "full": false,
                "lineColor": "rgb(31, 120, 193)",
                "show": false
              },
              "tableColumn": "",
              "targets": [
                {
                  "expr": "max without(instance,node) (\nkubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n/\nkubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100)\n",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "",
                  "refId": "A"
                }
              ],
              "thresholds": "80, 90",
              "title": "Volume inodes Usage",
              "tooltip": {
                "shared": false
              },
              "type": "singlestat",
              "valueFontSize": "80%",
              "valueMaps": [
                {
                  "op": "=",
                  "text": "N/A",
                  "value": "null"
                }
              ],
              "valueName": "current"
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        }
      ],
      "schemaVersion": 14,
      "style": "dark",
      "tags": [
        "kubernetes-mixin"
      ],
      "templating": {
        "list": [
          {
            "current": {
              "text": "default",
              "value": "default"
            },
            "hide": 0,
            "label": null,
            "name": "datasource",
            "options": [

            ],
            "query": "prometheus",
            "refresh": 1,
            "regex": "",
            "type": "datasource"
          },
          {
            "allValue": null,
            "current": {

            },
            "datasource": "$datasource",
            "hide": 2,
            "includeAll": false,
            "label": "cluster",
            "multi": false,
            "name": "cluster",
            "options": [

            ],
            "query": "label_values(kubelet_volume_stats_capacity_bytes, cluster)",
            "refresh": 2,
            "regex": "",
            "sort": 1,
            "tagValuesQuery": "",
            "tags": [

            ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
          },
          {
            "allValue": null,
            "current": {

            },
            "datasource": "$datasource",
            "hide": 0,
            "includeAll": false,
            "label": "Namespace",
            "multi": false,
            "name": "namespace",
            "options": [

            ],
            "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\"}, namespace)",
            "refresh": 2,
            "regex": "",
            "sort": 1,
            "tagValuesQuery": "",
            "tags": [

            ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
          },
          {
            "allValue": null,
            "current": {

            },
            "datasource": "$datasource",
            "hide": 0,
            "includeAll": false,
            "label": "PersistentVolumeClaim",
            "multi": false,
            "name": "volume",
            "options": [

            ],
            "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\"}, persistentvolumeclaim)",
            "refresh": 2,
            "regex": "",
            "sort": 1,
            "tagValuesQuery": "",
            "tags": [

            ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
          }
        ]
      },
      "time": {
        "from": "now-7d",
        "to": "now"
      },
      "timepicker": {
        "refresh_intervals": [
          "5s",
          "10s",
          "30s",
          "1m",
          "5m",
          "15m",
          "30m",
          "1h",
          "2h",
          "1d"
        ],
        "time_options": [
          "5m",
          "15m",
          "1h",
          "6h",
          "12h",
          "24h",
          "2d",
          "7d",
          "30d"
        ]
      },
      "timezone": "UTC",
      "title": "Kubernetes / Persistent Volumes",
      "uid": "919b92a8e8041bd567af9edab12c840c",
      "version": 0
    }
  scheduler.json: |-
    {
      "__inputs": [

      ],
      "__requires": [

      ],
      "annotations": {
        "list": [

        ]
      },
      "editable": false,
      "gnetId": null,
      "graphTooltip": 0,
      "hideControls": false,
      "id": null,
      "links": [

      ],
      "refresh": "10s",
      "rows": [
        {
          "collapse": false,
          "collapsed": false,
          "panels": [
            {
              "cacheTimeout": null,
              "colorBackground": false,
              "colorValue": false,
              "colors": [
                "#299c46",
                "rgba(237, 129, 40, 0.89)",
                "#d44a3a"
              ],
              "datasource": "$datasource",
              "format": "none",
              "gauge": {
                "maxValue": 100,
                "minValue": 0,
                "show": false,
                "thresholdLabels": false,
                "thresholdMarkers": true
              },
              "gridPos": {

              },
              "id": 2,
              "interval": null,
              "links": [

              ],
              "mappingType": 1,
              "mappingTypes": [
                {
                  "name": "value to text",
                  "value": 1
                },
                {
                  "name": "range to text",
                  "value": 2
                }
              ],
              "maxDataPoints": 100,
              "nullPointMode": "connected",
              "nullText": null,
              "postfix": "",
              "postfixFontSize": "50%",
              "prefix": "",
              "prefixFontSize": "50%",
              "rangeMaps": [
                {
                  "from": "null",
                  "text": "N/A",
                  "to": "null"
                }
              ],
              "span": 2,
              "sparkline": {
                "fillColor": "rgba(31, 118, 189, 0.18)",
                "full": false,
                "lineColor": "rgb(31, 120, 193)",
                "show": false
              },
              "tableColumn": "",
              "targets": [
                {
                  "expr": "sum(up{job=\"kube-scheduler\"})",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "",
                  "refId": "A"
                }
              ],
              "thresholds": "",
              "title": "Up",
              "tooltip": {
                "shared": false
              },
              "type": "singlestat",
              "valueFontSize": "80%",
              "valueMaps": [
                {
                  "op": "=",
                  "text": "N/A",
                  "value": "null"
                }
              ],
              "valueName": "min"
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 3,
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": true,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": true
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 5,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}} e2e",
                  "refId": "A"
                },
                {
                  "expr": "sum(rate(scheduler_binding_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}} binding",
                  "refId": "B"
                },
                {
                  "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}} scheduling algorithm",
                  "refId": "C"
                },
                {
                  "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}} volume",
                  "refId": "D"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Scheduling Rate",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "ops",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "ops",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 4,
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": true,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": true
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 5,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}} e2e",
                  "refId": "A"
                },
                {
                  "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}} binding",
                  "refId": "B"
                },
                {
                  "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}} scheduling algorithm",
                  "refId": "C"
                },
                {
                  "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}} volume",
                  "refId": "D"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Scheduling latency 99th Quantile",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        },
        {
          "collapse": false,
          "collapsed": false,
          "panels": [
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 5,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"2..\"}[5m]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "2xx",
                  "refId": "A"
                },
                {
                  "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"3..\"}[5m]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "3xx",
                  "refId": "B"
                },
                {
                  "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"4..\"}[5m]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "4xx",
                  "refId": "C"
                },
                {
                  "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"5..\"}[5m]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "5xx",
                  "refId": "D"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Kube API Request Rate",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "ops",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "ops",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 6,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 8,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"kube-scheduler\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{verb}} {{url}}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Post Request Latency 99th Quantile",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        },
        {
          "collapse": false,
          "collapsed": false,
          "panels": [
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 7,
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": true,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": true
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 12,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"kube-scheduler\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{verb}} {{url}}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Get Request Latency 99th Quantile",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        },
        {
          "collapse": false,
          "collapsed": false,
          "panels": [
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 8,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "process_resident_memory_bytes{job=\"kube-scheduler\", instance=~\"$instance\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Memory",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "bytes",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "bytes",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 9,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(process_cpu_seconds_total{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "CPU usage",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "bytes",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "bytes",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 10,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "go_goroutines{job=\"kube-scheduler\",instance=~\"$instance\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}}",
                  "refId": "A"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Goroutines",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        }
      ],
      "schemaVersion": 14,
      "style": "dark",
      "tags": [
        "kubernetes-mixin"
      ],
      "templating": {
        "list": [
          {
            "current": {
              "text": "default",
              "value": "default"
            },
            "hide": 0,
            "label": null,
            "name": "datasource",
            "options": [

            ],
            "query": "prometheus",
            "refresh": 1,
            "regex": "",
            "type": "datasource"
          },
          {
            "allValue": null,
            "current": {

            },
            "datasource": "$datasource",
            "hide": 0,
            "includeAll": true,
            "label": null,
            "multi": false,
            "name": "instance",
            "options": [

            ],
            "query": "label_values(process_cpu_seconds_total{job=\"kube-scheduler\"}, instance)",
            "refresh": 2,
            "regex": "",
            "sort": 1,
            "tagValuesQuery": "",
            "tags": [

            ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
          }
        ]
      },
      "time": {
        "from": "now-1h",
        "to": "now"
      },
      "timepicker": {
        "refresh_intervals": [
          "5s",
          "10s",
          "30s",
          "1m",
          "5m",
          "15m",
          "30m",
          "1h",
          "2h",
          "1d"
        ],
        "time_options": [
          "5m",
          "15m",
          "1h",
          "6h",
          "12h",
          "24h",
          "2d",
          "7d",
          "30d"
        ]
      },
      "timezone": "UTC",
      "title": "Kubernetes / Scheduler",
      "uid": "2e6b6a3b4bddf1427b3a55aa1311c656",
      "version": 0
    }
  statefulset.json: |-
    {
      "__inputs": [

      ],
      "__requires": [

      ],
      "annotations": {
        "list": [

        ]
      },
      "editable": false,
      "gnetId": null,
      "graphTooltip": 0,
      "hideControls": false,
      "id": null,
      "links": [

      ],
      "refresh": "",
      "rows": [
        {
          "collapse": false,
          "collapsed": false,
          "panels": [
            {
              "cacheTimeout": null,
              "colorBackground": false,
              "colorValue": false,
              "colors": [
                "#299c46",
                "rgba(237, 129, 40, 0.89)",
                "#d44a3a"
              ],
              "datasource": "$datasource",
              "format": "none",
              "gauge": {
                "maxValue": 100,
                "minValue": 0,
                "show": false,
                "thresholdLabels": false,
                "thresholdMarkers": true
              },
              "gridPos": {

              },
              "id": 2,
              "interval": null,
              "links": [

              ],
              "mappingType": 1,
              "mappingTypes": [
                {
                  "name": "value to text",
                  "value": 1
                },
                {
                  "name": "range to text",
                  "value": 2
                }
              ],
              "maxDataPoints": 100,
              "nullPointMode": "connected",
              "nullText": null,
              "postfix": "cores",
              "postfixFontSize": "50%",
              "prefix": "",
              "prefixFontSize": "50%",
              "rangeMaps": [
                {
                  "from": "null",
                  "text": "N/A",
                  "to": "null"
                }
              ],
              "span": 4,
              "sparkline": {
                "fillColor": "rgba(31, 118, 189, 0.18)",
                "lineColor": "rgb(31, 120, 193)",
                "show": true
              },
              "tableColumn": "",
              "targets": [
                {
                  "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", container!=\"\", namespace=\"$namespace\", pod=~\"$statefulset.*\"}[3m]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "",
                  "refId": "A"
                }
              ],
              "thresholds": "",
              "title": "CPU",
              "tooltip": {
                "shared": false
              },
              "type": "singlestat",
              "valueFontSize": "80%",
              "valueMaps": [
                {
                  "op": "=",
                  "text": "0",
                  "value": "null"
                }
              ],
              "valueName": "current"
            },
            {
              "cacheTimeout": null,
              "colorBackground": false,
              "colorValue": false,
              "colors": [
                "#299c46",
                "rgba(237, 129, 40, 0.89)",
                "#d44a3a"
              ],
              "datasource": "$datasource",
              "format": "none",
              "gauge": {
                "maxValue": 100,
                "minValue": 0,
                "show": false,
                "thresholdLabels": false,
                "thresholdMarkers": true
              },
              "gridPos": {

              },
              "id": 3,
              "interval": null,
              "links": [

              ],
              "mappingType": 1,
              "mappingTypes": [
                {
                  "name": "value to text",
                  "value": 1
                },
                {
                  "name": "range to text",
                  "value": 2
                }
              ],
              "maxDataPoints": 100,
              "nullPointMode": "connected",
              "nullText": null,
              "postfix": "GB",
              "postfixFontSize": "50%",
              "prefix": "",
              "prefixFontSize": "50%",
              "rangeMaps": [
                {
                  "from": "null",
                  "text": "N/A",
                  "to": "null"
                }
              ],
              "span": 4,
              "sparkline": {
                "fillColor": "rgba(31, 118, 189, 0.18)",
                "lineColor": "rgb(31, 120, 193)",
                "show": true
              },
              "tableColumn": "",
              "targets": [
                {
                  "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", container!=\"\", namespace=\"$namespace\", pod=~\"$statefulset.*\"}) / 1024^3",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "",
                  "refId": "A"
                }
              ],
              "thresholds": "",
              "title": "Memory",
              "tooltip": {
                "shared": false
              },
              "type": "singlestat",
              "valueFontSize": "80%",
              "valueMaps": [
                {
                  "op": "=",
                  "text": "0",
                  "value": "null"
                }
              ],
              "valueName": "current"
            },
            {
              "cacheTimeout": null,
              "colorBackground": false,
              "colorValue": false,
              "colors": [
                "#299c46",
                "rgba(237, 129, 40, 0.89)",
                "#d44a3a"
              ],
              "datasource": "$datasource",
              "format": "none",
              "gauge": {
                "maxValue": 100,
                "minValue": 0,
                "show": false,
                "thresholdLabels": false,
                "thresholdMarkers": true
              },
              "gridPos": {

              },
              "id": 4,
              "interval": null,
              "links": [

              ],
              "mappingType": 1,
              "mappingTypes": [
                {
                  "name": "value to text",
                  "value": 1
                },
                {
                  "name": "range to text",
                  "value": 2
                }
              ],
              "maxDataPoints": 100,
              "nullPointMode": "connected",
              "nullText": null,
              "postfix": "Bps",
              "postfixFontSize": "50%",
              "prefix": "",
              "prefixFontSize": "50%",
              "rangeMaps": [
                {
                  "from": "null",
                  "text": "N/A",
                  "to": "null"
                }
              ],
              "span": 4,
              "sparkline": {
                "fillColor": "rgba(31, 118, 189, 0.18)",
                "lineColor": "rgb(31, 120, 193)",
                "show": true
              },
              "tableColumn": "",
              "targets": [
                {
                  "expr": "sum(rate(container_network_transmit_bytes_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$statefulset.*\"}[3m])) + sum(rate(container_network_receive_bytes_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",pod=~\"$statefulset.*\"}[3m]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "",
                  "refId": "A"
                }
              ],
              "thresholds": "",
              "title": "Network",
              "tooltip": {
                "shared": false
              },
              "type": "singlestat",
              "valueFontSize": "80%",
              "valueMaps": [
                {
                  "op": "=",
                  "text": "0",
                  "value": "null"
                }
              ],
              "valueName": "current"
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        },
        {
          "collapse": false,
          "collapsed": false,
          "height": "100px",
          "panels": [
            {
              "cacheTimeout": null,
              "colorBackground": false,
              "colorValue": false,
              "colors": [
                "#299c46",
                "rgba(237, 129, 40, 0.89)",
                "#d44a3a"
              ],
              "datasource": "$datasource",
              "format": "none",
              "gauge": {
                "maxValue": 100,
                "minValue": 0,
                "show": false,
                "thresholdLabels": false,
                "thresholdMarkers": true
              },
              "gridPos": {

              },
              "id": 5,
              "interval": null,
              "links": [

              ],
              "mappingType": 1,
              "mappingTypes": [
                {
                  "name": "value to text",
                  "value": 1
                },
                {
                  "name": "range to text",
                  "value": 2
                }
              ],
              "maxDataPoints": 100,
              "nullPointMode": "connected",
              "nullText": null,
              "postfix": "",
              "postfixFontSize": "50%",
              "prefix": "",
              "prefixFontSize": "50%",
              "rangeMaps": [
                {
                  "from": "null",
                  "text": "N/A",
                  "to": "null"
                }
              ],
              "span": 3,
              "sparkline": {
                "fillColor": "rgba(31, 118, 189, 0.18)",
                "full": false,
                "lineColor": "rgb(31, 120, 193)",
                "show": false
              },
              "tableColumn": "",
              "targets": [
                {
                  "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "",
                  "refId": "A"
                }
              ],
              "thresholds": "",
              "title": "Desired Replicas",
              "tooltip": {
                "shared": false
              },
              "type": "singlestat",
              "valueFontSize": "80%",
              "valueMaps": [
                {
                  "op": "=",
                  "text": "0",
                  "value": "null"
                }
              ],
              "valueName": "current"
            },
            {
              "cacheTimeout": null,
              "colorBackground": false,
              "colorValue": false,
              "colors": [
                "#299c46",
                "rgba(237, 129, 40, 0.89)",
                "#d44a3a"
              ],
              "datasource": "$datasource",
              "format": "none",
              "gauge": {
                "maxValue": 100,
                "minValue": 0,
                "show": false,
                "thresholdLabels": false,
                "thresholdMarkers": true
              },
              "gridPos": {

              },
              "id": 6,
              "interval": null,
              "links": [

              ],
              "mappingType": 1,
              "mappingTypes": [
                {
                  "name": "value to text",
                  "value": 1
                },
                {
                  "name": "range to text",
                  "value": 2
                }
              ],
              "maxDataPoints": 100,
              "nullPointMode": "connected",
              "nullText": null,
              "postfix": "",
              "postfixFontSize": "50%",
              "prefix": "",
              "prefixFontSize": "50%",
              "rangeMaps": [
                {
                  "from": "null",
                  "text": "N/A",
                  "to": "null"
                }
              ],
              "span": 3,
              "sparkline": {
                "fillColor": "rgba(31, 118, 189, 0.18)",
                "full": false,
                "lineColor": "rgb(31, 120, 193)",
                "show": false
              },
              "tableColumn": "",
              "targets": [
                {
                  "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "",
                  "refId": "A"
                }
              ],
              "thresholds": "",
              "title": "Replicas of current version",
              "tooltip": {
                "shared": false
              },
              "type": "singlestat",
              "valueFontSize": "80%",
              "valueMaps": [
                {
                  "op": "=",
                  "text": "0",
                  "value": "null"
                }
              ],
              "valueName": "current"
            },
            {
              "cacheTimeout": null,
              "colorBackground": false,
              "colorValue": false,
              "colors": [
                "#299c46",
                "rgba(237, 129, 40, 0.89)",
                "#d44a3a"
              ],
              "datasource": "$datasource",
              "format": "none",
              "gauge": {
                "maxValue": 100,
                "minValue": 0,
                "show": false,
                "thresholdLabels": false,
                "thresholdMarkers": true
              },
              "gridPos": {

              },
              "id": 7,
              "interval": null,
              "links": [

              ],
              "mappingType": 1,
              "mappingTypes": [
                {
                  "name": "value to text",
                  "value": 1
                },
                {
                  "name": "range to text",
                  "value": 2
                }
              ],
              "maxDataPoints": 100,
              "nullPointMode": "connected",
              "nullText": null,
              "postfix": "",
              "postfixFontSize": "50%",
              "prefix": "",
              "prefixFontSize": "50%",
              "rangeMaps": [
                {
                  "from": "null",
                  "text": "N/A",
                  "to": "null"
                }
              ],
              "span": 3,
              "sparkline": {
                "fillColor": "rgba(31, 118, 189, 0.18)",
                "full": false,
                "lineColor": "rgb(31, 120, 193)",
                "show": false
              },
              "tableColumn": "",
              "targets": [
                {
                  "expr": "max(kube_statefulset_status_observed_generation{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "",
                  "refId": "A"
                }
              ],
              "thresholds": "",
              "title": "Observed Generation",
              "tooltip": {
                "shared": false
              },
              "type": "singlestat",
              "valueFontSize": "80%",
              "valueMaps": [
                {
                  "op": "=",
                  "text": "0",
                  "value": "null"
                }
              ],
              "valueName": "current"
            },
            {
              "cacheTimeout": null,
              "colorBackground": false,
              "colorValue": false,
              "colors": [
                "#299c46",
                "rgba(237, 129, 40, 0.89)",
                "#d44a3a"
              ],
              "datasource": "$datasource",
              "format": "none",
              "gauge": {
                "maxValue": 100,
                "minValue": 0,
                "show": false,
                "thresholdLabels": false,
                "thresholdMarkers": true
              },
              "gridPos": {

              },
              "id": 8,
              "interval": null,
              "links": [

              ],
              "mappingType": 1,
              "mappingTypes": [
                {
                  "name": "value to text",
                  "value": 1
                },
                {
                  "name": "range to text",
                  "value": 2
                }
              ],
              "maxDataPoints": 100,
              "nullPointMode": "connected",
              "nullText": null,
              "postfix": "",
              "postfixFontSize": "50%",
              "prefix": "",
              "prefixFontSize": "50%",
              "rangeMaps": [
                {
                  "from": "null",
                  "text": "N/A",
                  "to": "null"
                }
              ],
              "span": 3,
              "sparkline": {
                "fillColor": "rgba(31, 118, 189, 0.18)",
                "full": false,
                "lineColor": "rgb(31, 120, 193)",
                "show": false
              },
              "tableColumn": "",
              "targets": [
                {
                  "expr": "max(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "",
                  "refId": "A"
                }
              ],
              "thresholds": "",
              "title": "Metadata Generation",
              "tooltip": {
                "shared": false
              },
              "type": "singlestat",
              "valueFontSize": "80%",
              "valueMaps": [
                {
                  "op": "=",
                  "text": "0",
                  "value": "null"
                }
              ],
              "valueName": "current"
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        },
        {
          "collapse": false,
          "collapsed": false,
          "panels": [
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {

              },
              "id": 9,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "sideWidth": null,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": null,
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "replicas specified",
                  "refId": "A"
                },
                {
                  "expr": "max(kube_statefulset_status_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "replicas created",
                  "refId": "B"
                },
                {
                  "expr": "min(kube_statefulset_status_replicas_ready{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "ready",
                  "refId": "C"
                },
                {
                  "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "replicas of current version",
                  "refId": "D"
                },
                {
                  "expr": "min(kube_statefulset_status_replicas_updated{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "updated",
                  "refId": "E"
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Replicas",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": false,
          "title": "Dashboard Row",
          "titleSize": "h6",
          "type": "row"
        }
      ],
      "schemaVersion": 14,
      "style": "dark",
      "tags": [
        "kubernetes-mixin"
      ],
      "templating": {
        "list": [
          {
            "current": {
              "text": "default",
              "value": "default"
            },
            "hide": 0,
            "label": null,
            "name": "datasource",
            "options": [

            ],
            "query": "prometheus",
            "refresh": 1,
            "regex": "",
            "type": "datasource"
          },
          {
            "allValue": null,
            "current": {

            },
            "datasource": "$datasource",
            "hide": 2,
            "includeAll": false,
            "label": "cluster",
            "multi": false,
            "name": "cluster",
            "options": [

            ],
            "query": "label_values(kube_statefulset_metadata_generation, cluster)",
            "refresh": 2,
            "regex": "",
            "sort": 1,
            "tagValuesQuery": "",
            "tags": [

            ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
          },
          {
            "allValue": null,
            "current": {

            },
            "datasource": "$datasource",
            "hide": 0,
            "includeAll": false,
            "label": "Namespace",
            "multi": false,
            "name": "namespace",
            "options": [

            ],
            "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)",
            "refresh": 2,
            "regex": "",
            "sort": 1,
            "tagValuesQuery": "",
            "tags": [

            ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
          },
          {
            "allValue": null,
            "current": {

            },
            "datasource": "$datasource",
            "hide": 0,
            "includeAll": false,
            "label": "Name",
            "multi": false,
            "name": "statefulset",
            "options": [

            ],
            "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, statefulset)",
            "refresh": 2,
            "regex": "",
            "sort": 1,
            "tagValuesQuery": "",
            "tags": [

            ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
          }
        ]
      },
      "time": {
        "from": "now-1h",
        "to": "now"
      },
      "timepicker": {
        "refresh_intervals": [
          "5s",
          "10s",
          "30s",
          "1m",
          "5m",
          "15m",
          "30m",
          "1h",
          "2h",
          "1d"
        ],
        "time_options": [
          "5m",
          "15m",
          "1h",
          "6h",
          "12h",
          "24h",
          "2d",
          "7d",
          "30d"
        ]
      },
      "timezone": "UTC",
      "title": "Kubernetes / StatefulSets",
      "uid": "a31c1f46e6f727cb37c0d731a7245005",
      "version": 0
    }
kind: ConfigMap
metadata:
  name: grafana-dashboards-k8s
  namespace: monitoring