Skip to main content

How to scrape selective kube state metrics

This document describes how to scrape selective kube state metrics

Obtain the list of kube state metrics from your levitate cluster​

curl -XGET 'https://<username>:<password>@read-app-tsdb.last9.io/hot/v1/metrics/<token>/sender/<org_slug>/api/v1/label/__name__/values' | jq | grep -e "kube_"

"kube_apiserver_pod_logs_pods_logs_backend_tls_failure_total",
"kube_apiserver_pod_logs_pods_logs_insecure_backend_total",
"kube_certificatesigningrequest_annotations",
"kube_certificatesigningrequest_cert_length",
"kube_certificatesigningrequest_condition",
"kube_certificatesigningrequest_created",
"kube_certificatesigningrequest_labels",
"kube_configmap_annotations",
"kube_configmap_created",
"kube_configmap_info",
"kube_configmap_labels",
"kube_configmap_metadata_resource_version",
"kube_daemonset_annotations",
"kube_daemonset_created",
"kube_daemonset_labels",
"kube_daemonset_metadata_generation",
"kube_daemonset_status_current_number_scheduled",
"kube_daemonset_status_desired_number_scheduled",
"kube_daemonset_status_number_available",
"kube_daemonset_status_number_misscheduled",
"kube_daemonset_status_number_ready",
"kube_daemonset_status_number_unavailable",
"kube_daemonset_status_observed_generation",
"kube_daemonset_status_updated_number_scheduled",
"kube_deployment_annotations",
"kube_deployment_created",
"kube_deployment_labels",
"kube_deployment_metadata_generation",
"kube_deployment_spec_paused",
"kube_deployment_spec_replicas",
"kube_deployment_spec_strategy_rollingupdate_max_surge",
"kube_deployment_spec_strategy_rollingupdate_max_unavailable",
"kube_deployment_status_condition",
"kube_deployment_status_observed_generation",
"kube_deployment_status_replicas",
"kube_deployment_status_replicas_available",
"kube_deployment_status_replicas_ready",
"kube_deployment_status_replicas_unavailable",
"kube_deployment_status_replicas_updated",
"kube_endpoint_address",
"kube_endpoint_address_available",
"kube_endpoint_address_not_ready",
"kube_endpoint_annotations",
"kube_endpoint_created",
"kube_endpoint_info",
"kube_endpoint_labels",
"kube_endpoint_ports",
"kube_ingress_annotations",
"kube_ingress_created",
"kube_ingress_info",
"kube_ingress_labels",
"kube_ingress_metadata_resource_version",
"kube_ingress_path",
"kube_job_annotations",
"kube_job_complete",
"kube_job_created",
"kube_job_info",
"kube_job_labels",
"kube_job_owner",
"kube_job_spec_completions",
"kube_job_spec_parallelism",
"kube_job_status_active",
"kube_job_status_completion_time",
"kube_job_status_failed",
"kube_job_status_start_time",
"kube_job_status_succeeded",
"kube_lease_owner",
"kube_lease_renew_time",
"kube_mutatingwebhookconfiguration_created",
"kube_mutatingwebhookconfiguration_info",
"kube_mutatingwebhookconfiguration_metadata_resource_version",
"kube_namespace_annotations",
"kube_namespace_created",
"kube_namespace_labels",
"kube_namespace_status_phase",
"kube_node_annotations",
"kube_node_created",
"kube_node_deletion_timestamp",
"kube_node_info",
"kube_node_labels",
"kube_node_spec_taint",
"kube_node_spec_unschedulable",
"kube_node_status_allocatable",
"kube_node_status_capacity",
"kube_node_status_condition",
"kube_persistentvolume_annotations",
"kube_persistentvolume_capacity_bytes",
"kube_persistentvolume_claim_ref",
"kube_persistentvolume_created",
"kube_persistentvolume_info",
"kube_persistentvolume_labels",
"kube_persistentvolume_status_phase",
"kube_persistentvolumeclaim_access_mode",
"kube_persistentvolumeclaim_annotations",
"kube_persistentvolumeclaim_created",
"kube_persistentvolumeclaim_info",
"kube_persistentvolumeclaim_labels",
"kube_persistentvolumeclaim_resource_requests_storage_bytes",
"kube_persistentvolumeclaim_status_phase",
"kube_pod_annotations",
"kube_pod_completion_time",
"kube_pod_container_info",
"kube_pod_container_resource_limits",
"kube_pod_container_resource_requests",
"kube_pod_container_state_started",
"kube_pod_container_status_last_terminated_exitcode",
"kube_pod_container_status_last_terminated_reason",
"kube_pod_container_status_ready",
"kube_pod_container_status_restarts_total",
"kube_pod_container_status_running",
"kube_pod_container_status_terminated",
"kube_pod_container_status_terminated_reason",
"kube_pod_container_status_waiting",
"kube_pod_container_status_waiting_reason",
"kube_pod_created",
"kube_pod_deletion_timestamp",
"kube_pod_info",
"kube_pod_init_container_info",
"kube_pod_init_container_status_ready",
"kube_pod_init_container_status_restarts_total",
"kube_pod_init_container_status_running",
"kube_pod_init_container_status_terminated",
"kube_pod_init_container_status_terminated_reason",
"kube_pod_init_container_status_waiting",
"kube_pod_init_container_status_waiting_reason",
"kube_pod_ips",
"kube_pod_labels",
"kube_pod_owner",
"kube_pod_restart_policy",
"kube_pod_spec_volumes_persistentvolumeclaims_info",
"kube_pod_spec_volumes_persistentvolumeclaims_readonly",
"kube_pod_start_time",
"kube_pod_status_container_ready_time",
"kube_pod_status_phase",
"kube_pod_status_qos_class",
"kube_pod_status_ready",
"kube_pod_status_ready_time",
"kube_pod_status_reason",
"kube_pod_status_scheduled",
"kube_pod_status_scheduled_time",
"kube_pod_status_unschedulable",
"kube_pod_tolerations",
"kube_poddisruptionbudget_annotations",
"kube_poddisruptionbudget_created",
"kube_poddisruptionbudget_labels",
"kube_poddisruptionbudget_status_current_healthy",
"kube_poddisruptionbudget_status_desired_healthy",
"kube_poddisruptionbudget_status_expected_pods",
"kube_poddisruptionbudget_status_observed_generation",
"kube_poddisruptionbudget_status_pod_disruptions_allowed",
"kube_replicaset_annotations",
"kube_replicaset_created",
"kube_replicaset_labels",
"kube_replicaset_metadata_generation",
"kube_replicaset_owner",
"kube_replicaset_spec_replicas",
"kube_replicaset_status_fully_labeled_replicas",
"kube_replicaset_status_observed_generation",
"kube_replicaset_status_ready_replicas",
"kube_replicaset_status_replicas",
"kube_secret_annotations",
"kube_secret_created",
"kube_secret_info",
"kube_secret_labels",
"kube_secret_metadata_resource_version",
"kube_secret_type",
"kube_service_annotations",
"kube_service_created",
"kube_service_info",
"kube_service_labels",
"kube_service_spec_type",
"kube_service_status_load_balancer_ingress",
"kube_storageclass_annotations",
"kube_storageclass_created",
"kube_storageclass_info",
"kube_storageclass_labels",
"kube_validatingwebhookconfiguration_created",
"kube_validatingwebhookconfiguration_info",
"kube_validatingwebhookconfiguration_metadata_resource_version",

Let’s decide to omit kube_certificatesigningrequest_* metrics​

  1. Prepare a list of metrics you want to omit. This needs to be a comma separated array of strings.

    [
      "kube_certificatesigningrequest_annotations",
      "kube_certificatesigningrequest_cert_length",
      "kube_certificatesigningrequest_condition",
      "kube_certificatesigningrequest_created",
      "kube_certificatesigningrequest_labels"
    ]

  2. Find the metricDenylist configuration in your Kube State Metrics Helm chart and append this list to that config.

    metricDenylist: ["kube_certificatesigningrequest_annotations", "kube_certificatesigningrequest_cert_length", "kube_certificatesigningrequest_condition", "kube_certificatesigningrequest_created", "kube_certificatesigningrequest_labels"]

  3. Now deploy your Kube State Metrics Helm chart as usual.

  4. Run the below command after the deployment to verify that it is in effect. You should not find any metrics with kube_certificatesigningrequest prefix being emitted anymore.

curl -XGET 'https://<username>:<password>@read-app-tsdb.last9.io/hot/v1/metrics/<token>/sender/<org_slug>/api/v1/label/__name__/values' | jq | grep -e "kube_certificatesigningrequest_"