From 5a0d6d639b305f0b1562bfeff1b1682826409ffb Mon Sep 17 00:00:00 2001
From: royalcat <k.adamovich20@gmail.com>
Date: Mon, 3 Mar 2025 16:47:59 +0300
Subject: [PATCH] storage and metrics

---
 longhorn-system/helm-create-deploy.sh         |    1 +
 longhorn-system/helm-values.yaml              |  536 ++
 longhorn-system/longhorn.yaml                 | 5421 +++++++++++++++++
 longhorn-system/minio-backupstore.yaml        |   91 +
 metrics/grafana-configmap.yaml                |   60 +
 .../grafana-data-persistentvolumeclaim.yaml   |   13 +
 metrics/grafana-deployment.yaml               |   84 +
 metrics/loki-configmap.yaml                   |   73 +
 metrics/loki-data-persistentvolumeclaim.yaml  |   13 +
 metrics/loki-deployment.yaml                  |   49 +
 metrics/loki-service.yaml                     |   14 +
 metrics/otel-collector-configmap.yaml         |   76 +
 metrics/otel-collector-deployment.yaml        |   47 +
 metrics/otel-collector-service.yaml           |   17 +
 metrics/prometheus.yaml                       |  172 +
 metrics/pyroscope-configmap.yaml              |   18 +
 .../pyroscope-data-persistentvolumeclaim.yaml |   14 +
 metrics/pyroscope-deployment.yaml             |   48 +
 metrics/pyroscope-service.yaml                |   14 +
 metrics/tempo-configmap.yaml                  |   44 +
 metrics/tempo-data-persistentvolumeclaim.yaml |   13 +
 metrics/tempo-deployment.yaml                 |   54 +
 metrics/tempo-service.yaml                    |   20 +
 23 files changed, 6892 insertions(+)
 create mode 100644 longhorn-system/helm-create-deploy.sh
 create mode 100644 longhorn-system/helm-values.yaml
 create mode 100644 longhorn-system/longhorn.yaml
 create mode 100644 longhorn-system/minio-backupstore.yaml
 create mode 100644 metrics/grafana-configmap.yaml
 create mode 100644 metrics/grafana-data-persistentvolumeclaim.yaml
 create mode 100644 metrics/grafana-deployment.yaml
 create mode 100644 metrics/loki-configmap.yaml
 create mode 100644 metrics/loki-data-persistentvolumeclaim.yaml
 create mode 100644 metrics/loki-deployment.yaml
 create mode 100644 metrics/loki-service.yaml
 create mode 100644 metrics/otel-collector-configmap.yaml
 create mode 100644 metrics/otel-collector-deployment.yaml
 create mode 100644 metrics/otel-collector-service.yaml
 create mode 100644 metrics/prometheus.yaml
 create mode 100644 metrics/pyroscope-configmap.yaml
 create mode 100644 metrics/pyroscope-data-persistentvolumeclaim.yaml
 create mode 100644 metrics/pyroscope-deployment.yaml
 create mode 100644 metrics/pyroscope-service.yaml
 create mode 100644 metrics/tempo-configmap.yaml
 create mode 100644 metrics/tempo-data-persistentvolumeclaim.yaml
 create mode 100644 metrics/tempo-deployment.yaml
 create mode 100644 metrics/tempo-service.yaml

diff --git a/longhorn-system/helm-create-deploy.sh b/longhorn-system/helm-create-deploy.sh
new file mode 100644
index 0000000..2b895df
--- /dev/null
+++ b/longhorn-system/helm-create-deploy.sh
@@ -0,0 +1 @@
+helm template -f helm-values.yaml longhorn longhorn/longhorn --namespace longhorn-system --create-namespace --version 1.8.0 > longhorn.yaml
\ No newline at end of file
diff --git a/longhorn-system/helm-values.yaml b/longhorn-system/helm-values.yaml
new file mode 100644
index 0000000..ba43a31
--- /dev/null
+++ b/longhorn-system/helm-values.yaml
@@ -0,0 +1,536 @@
+# Declare variables to be passed into your templates.
+global:
+  # -- Toleration for nodes allowed to run user-deployed components such as Longhorn Manager, Longhorn UI, and Longhorn Driver Deployer.
+  tolerations: []
+  # -- Node selector for nodes allowed to run user-deployed components such as Longhorn Manager, Longhorn UI, and Longhorn Driver Deployer.
+  nodeSelector: {}
+  cattle:
+    # -- Default system registry.
+    systemDefaultRegistry: ""
+    windowsCluster:
+      # -- Setting that allows Longhorn to run on a Rancher Windows cluster.
+      enabled: false
+      # -- Toleration for Linux nodes that can run user-deployed Longhorn components.
+      tolerations:
+        - key: "cattle.io/os"
+          value: "linux"
+          effect: "NoSchedule"
+          operator: "Equal"
+      # -- Node selector for Linux nodes that can run user-deployed Longhorn components.
+      nodeSelector:
+        kubernetes.io/os: "linux"
+      defaultSetting:
+        # -- Toleration for system-managed Longhorn components.
+        taintToleration: cattle.io/os=linux:NoSchedule
+        # -- Node selector for system-managed Longhorn components.
+        systemManagedComponentsNodeSelector: kubernetes.io/os:linux
+
+networkPolicies:
+  # -- Setting that allows you to enable network policies that control access to Longhorn pods.
+  enabled: false
+  # -- Distribution that determines the policy for allowing access for an ingress. (Options: "k3s", "rke2", "rke1")
+  type: "k3s"
+
+image:
+  longhorn:
+    engine:
+      # -- Repository for the Longhorn Engine image.
+      repository: longhornio/longhorn-engine
+      # -- Tag for the Longhorn Engine image.
+      tag: v1.8.0
+    manager:
+      # -- Repository for the Longhorn Manager image.
+      repository: longhornio/longhorn-manager
+      # -- Tag for the Longhorn Manager image.
+      tag: v1.8.0
+    ui:
+      # -- Repository for the Longhorn UI image.
+      repository: longhornio/longhorn-ui
+      # -- Tag for the Longhorn UI image.
+      tag: v1.8.0
+    instanceManager:
+      # -- Repository for the Longhorn Instance Manager image.
+      repository: longhornio/longhorn-instance-manager
+      # -- Tag for the Longhorn Instance Manager image.
+      tag: v1.8.0
+    shareManager:
+      # -- Repository for the Longhorn Share Manager image.
+      repository: longhornio/longhorn-share-manager
+      # -- Tag for the Longhorn Share Manager image.
+      tag: v1.8.0
+    backingImageManager:
+      # -- Repository for the Backing Image Manager image. When unspecified, Longhorn uses the default value.
+      repository: longhornio/backing-image-manager
+      # -- Tag for the Backing Image Manager image. When unspecified, Longhorn uses the default value.
+      tag: v1.8.0
+    supportBundleKit:
+      # -- Repository for the Longhorn Support Bundle Manager image.
+      repository: longhornio/support-bundle-kit
+      # -- Tag for the Longhorn Support Bundle Manager image.
+      tag: v0.0.49
+  csi:
+    attacher:
+      # -- Repository for the CSI attacher image. When unspecified, Longhorn uses the default value.
+      repository: longhornio/csi-attacher
+      # -- Tag for the CSI attacher image. When unspecified, Longhorn uses the default value.
+      tag: v4.8.0
+    provisioner:
+      # -- Repository for the CSI Provisioner image. When unspecified, Longhorn uses the default value.
+      repository: longhornio/csi-provisioner
+      # -- Tag for the CSI Provisioner image. When unspecified, Longhorn uses the default value.
+      tag: v5.1.0-20241220
+    nodeDriverRegistrar:
+      # -- Repository for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value.
+      repository: longhornio/csi-node-driver-registrar
+      # -- Tag for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value.
+      tag: v2.13.0
+    resizer:
+      # -- Repository for the CSI Resizer image. When unspecified, Longhorn uses the default value.
+      repository: longhornio/csi-resizer
+      # -- Tag for the CSI Resizer image. When unspecified, Longhorn uses the default value.
+      tag: v1.13.1
+    snapshotter:
+      # -- Repository for the CSI Snapshotter image. When unspecified, Longhorn uses the default value.
+      repository: longhornio/csi-snapshotter
+      # -- Tag for the CSI Snapshotter image. When unspecified, Longhorn uses the default value.
+      tag: v8.2.0
+    livenessProbe:
+      # -- Repository for the CSI liveness probe image. When unspecified, Longhorn uses the default value.
+      repository: longhornio/livenessprobe
+      # -- Tag for the CSI liveness probe image. When unspecified, Longhorn uses the default value.
+      tag: v2.15.0
+  openshift:
+    oauthProxy:
+      # -- Repository for the OAuth Proxy image. Specify the upstream image (for example, "quay.io/openshift/origin-oauth-proxy"). This setting applies only to OpenShift users.
+      repository: ""
+      # -- Tag for the OAuth Proxy image. Specify OCP/OKD version 4.1 or later (including version 4.15, which is available at quay.io/openshift/origin-oauth-proxy:4.15). This setting applies only to OpenShift users.
+      tag: ""
+  # -- Image pull policy that applies to all user-deployed Longhorn components, such as Longhorn Manager, Longhorn driver, and Longhorn UI.
+  pullPolicy: IfNotPresent
+
+service:
+  ui:
+    # -- Service type for Longhorn UI. (Options: "ClusterIP", "NodePort", "LoadBalancer", "Rancher-Proxy")
+    type: ClusterIP
+    # -- NodePort port number for Longhorn UI. When unspecified, Longhorn selects a free port between 30000 and 32767.
+    nodePort: null
+  manager:
+    # -- Service type for Longhorn Manager.
+    type: ClusterIP
+    # -- NodePort port number for Longhorn Manager. When unspecified, Longhorn selects a free port between 30000 and 32767.
+    nodePort: ""
+
+persistence:
+  # -- Setting that allows you to specify the default Longhorn StorageClass.
+  defaultClass: true
+  # -- Filesystem type of the default Longhorn StorageClass.
+  defaultFsType: ext4
+  # -- mkfs parameters of the default Longhorn StorageClass.
+  defaultMkfsParams: ""
+  # -- Replica count of the default Longhorn StorageClass.
+  defaultClassReplicaCount: 1
+  # -- Data locality of the default Longhorn StorageClass. (Options: "disabled", "best-effort")
+  defaultDataLocality: best-effort
+  # -- Reclaim policy that provides instructions for handling of a volume after its claim is released. (Options: "Retain", "Delete")
+  reclaimPolicy: Delete
+  # -- Setting that allows you to enable live migration of a Longhorn volume from one node to another.
+  migratable: false
+  # -- Setting that disables the revision counter and thereby prevents Longhorn from tracking all write operations to a volume. When salvaging a volume, Longhorn uses properties of the volume-head-xxx.img file (the last file size and the last time the file was modified) to select the replica to be used for volume recovery.
+  disableRevisionCounter: "true"
+  # -- Set NFS mount options for Longhorn StorageClass for RWX volumes
+  nfsOptions: ""
+  recurringJobSelector:
+    # -- Setting that allows you to enable the recurring job selector for a Longhorn StorageClass.
+    enable: false
+    # -- Recurring job selector for a Longhorn StorageClass. Ensure that quotes are used correctly when specifying job parameters. (Example: `[{"name":"backup", "isGroup":true}]`)
+    jobList: []
+  backingImage:
+    # -- Setting that allows you to use a backing image in a Longhorn StorageClass.
+    enable: false
+    # -- Backing image to be used for creating and restoring volumes in a Longhorn StorageClass. When no backing images are available, specify the data source type and parameters that Longhorn can use to create a backing image.
+    name: ~
+    # -- Data source type of a backing image used in a Longhorn StorageClass.
+    # If the backing image exists in the cluster, Longhorn uses this setting to verify the image.
+    # If the backing image does not exist, Longhorn creates one using the specified data source type.
+    dataSourceType: ~
+    # -- Data source parameters of a backing image used in a Longhorn StorageClass.
+    # You can specify a JSON string of a map. (Example: `'{\"url\":\"https://backing-image-example.s3-region.amazonaws.com/test-backing-image\"}'`)
+    dataSourceParameters: ~
+    # -- Expected SHA-512 checksum of a backing image used in a Longhorn StorageClass.
+    expectedChecksum: ~
+  defaultDiskSelector:
+    # -- Setting that allows you to enable the disk selector for the default Longhorn StorageClass.
+    enable: false
+    # -- Disk selector for the default Longhorn StorageClass. Longhorn uses only disks with the specified tags for storing volume data. (Examples: "nvme,sata")
+    selector: ""
+  defaultNodeSelector:
+    # -- Setting that allows you to enable the node selector for the default Longhorn StorageClass.
+    enable: false
+    # -- Node selector for the default Longhorn StorageClass. Longhorn uses only nodes with the specified tags for storing volume data. (Examples: "storage,fast")
+    selector: ""
+  # -- Setting that allows you to enable automatic snapshot removal during filesystem trim for a Longhorn StorageClass. (Options: "ignored", "enabled", "disabled")
+  removeSnapshotsDuringFilesystemTrim: ignored
+  # -- Setting that allows you to specify the data engine version for the default Longhorn StorageClass. (Options: "v1", "v2")
+  dataEngine: v1
+  # -- Setting that allows you to specify the backup target for the default Longhorn StorageClass.
+  backupTargetName: default
+
+preUpgradeChecker:
+  # -- Setting that allows Longhorn to perform pre-upgrade checks. Disable this setting when installing Longhorn using Argo CD or other GitOps solutions.
+  jobEnabled: true
+  # -- Setting that allows Longhorn to perform upgrade version checks after starting the Longhorn Manager DaemonSet Pods. Disabling this setting also disables `preUpgradeChecker.jobEnabled`. Longhorn recommends keeping this setting enabled.
+  upgradeVersionCheck: true
+
+csi:
+  # -- kubelet root directory. When unspecified, Longhorn uses the default value.
+  kubeletRootDir: ~
+  # -- Replica count of the CSI Attacher. When unspecified, Longhorn uses the default value ("3").
+  attacherReplicaCount: 1
+  # -- Replica count of the CSI Provisioner. When unspecified, Longhorn uses the default value ("3").
+  provisionerReplicaCount: 1
+  # -- Replica count of the CSI Resizer. When unspecified, Longhorn uses the default value ("3").
+  resizerReplicaCount: 1
+  # -- Replica count of the CSI Snapshotter. When unspecified, Longhorn uses the default value ("3").
+  snapshotterReplicaCount: 1
+
+defaultSettings:
+  # -- Setting that allows Longhorn to automatically attach a volume and create snapshots or backups when recurring jobs are run.
+  allowRecurringJobWhileVolumeDetached: ~
+  # -- Setting that allows Longhorn to automatically create a default disk only on nodes with the label "node.longhorn.io/create-default-disk=true" (if no other disks exist). When this setting is disabled, Longhorn creates a default disk on each node that is added to the cluster.
+  createDefaultDiskLabeledNodes: ~
+  # -- Default path for storing data on a host. The default value is "/var/lib/longhorn/".
+  defaultDataPath: "/var/lib/longhorn/"
+  # -- Default data locality. A Longhorn volume has data locality if a local replica of the volume exists on the same node as the pod that is using the volume.
+  defaultDataLocality: best-effort
+  # -- Setting that allows scheduling on nodes with healthy replicas of the same volume. This setting is disabled by default.
+  replicaSoftAntiAffinity: ~
+  # -- Setting that automatically rebalances replicas when an available node is discovered.
+  replicaAutoBalance: ~
+  # -- Percentage of storage that can be allocated relative to hard drive capacity. The default value is "100".
+  storageOverProvisioningPercentage: ~
+  # -- Percentage of minimum available disk capacity. When the minimum available capacity exceeds the total available capacity, the disk becomes unschedulable until more space is made available for use. The default value is "25".
+  storageMinimalAvailablePercentage: 10
+  # -- Percentage of disk space that is not allocated to the default disk on each new Longhorn node.
+  storageReservedPercentageForDefaultDisk: 10
+  # -- Upgrade Checker that periodically checks for new Longhorn versions. When a new version is available, a notification appears on the Longhorn UI. This setting is enabled by default
+  upgradeChecker: ~
+  # -- Default number of replicas for volumes created using the Longhorn UI. For Kubernetes configuration, modify the `numberOfReplicas` field in the StorageClass. The default value is "3".
+  defaultReplicaCount: 1
+  # -- Default name of Longhorn static StorageClass. "storageClassName" is assigned to PVs and PVCs that are created for an existing Longhorn volume. "storageClassName" can also be used as a label, so it is possible to use a Longhorn StorageClass to bind a workload to an existing PV without creating a Kubernetes StorageClass object. "storageClassName" needs to be an existing StorageClass. The default value is "longhorn-static".
+  defaultLonghornStaticStorageClass: ~
+  # -- Number of minutes that Longhorn keeps a failed backup resource. When the value is "0", automatic deletion is disabled.
+  failedBackupTTL: ~
+  # -- Number of minutes that Longhorn allows for the backup execution. The default value is "1".
+  backupExecutionTimeout: ~
+  # -- Setting that restores recurring jobs from a backup volume on a backup target and creates recurring jobs if none exist during backup restoration.
+  restoreVolumeRecurringJobs: ~
+  # -- Maximum number of successful recurring backup and snapshot jobs to be retained. When the value is "0", a history of successful recurring jobs is not retained.
+  recurringSuccessfulJobsHistoryLimit: ~
+  # -- Maximum number of failed recurring backup and snapshot jobs to be retained. When the value is "0", a history of failed recurring jobs is not retained.
+  recurringFailedJobsHistoryLimit: ~
+  # -- Maximum number of snapshots or backups to be retained.
+  recurringJobMaxRetention: ~
+  # -- Maximum number of failed support bundles that can exist in the cluster. When the value is "0", Longhorn automatically purges all failed support bundles.
+  supportBundleFailedHistoryLimit: ~
+  # -- Taint or toleration for system-managed Longhorn components.
+  # Specify values using a semicolon-separated list in `kubectl taint` syntax (Example: key1=value1:effect; key2=value2:effect).
+  taintToleration: ~
+  # -- Node selector for system-managed Longhorn components.
+  systemManagedComponentsNodeSelector: ~
+  # -- PriorityClass for system-managed Longhorn components.
+  # This setting can help prevent Longhorn components from being evicted under Node Pressure.
+  # Notice that this will be applied to Longhorn user-deployed components by default if there are no priority class values set yet, such as `longhornManager.priorityClass`.
+  priorityClass: &defaultPriorityClassNameRef "longhorn-critical"
+  # -- Setting that allows Longhorn to automatically salvage volumes when all replicas become faulty (for example, when the network connection is interrupted). Longhorn determines which replicas are usable and then uses these replicas for the volume. This setting is enabled by default.
+  autoSalvage: ~
+  # -- Setting that allows Longhorn to automatically delete a workload pod that is managed by a controller (for example, daemonset) whenever a Longhorn volume is detached unexpectedly (for example, during Kubernetes upgrades). After deletion, the controller restarts the pod and then Kubernetes handles volume reattachment and remounting.
+  autoDeletePodWhenVolumeDetachedUnexpectedly: ~
+  # -- Setting that prevents Longhorn Manager from scheduling replicas on a cordoned Kubernetes node. This setting is enabled by default.
+  disableSchedulingOnCordonedNode: ~
+  # -- Setting that allows Longhorn to schedule new replicas of a volume to nodes in the same zone as existing healthy replicas. Nodes that do not belong to any zone are treated as existing in the zone that contains healthy replicas. When identifying zones, Longhorn relies on the label "topology.kubernetes.io/zone=<Zone name of the node>" in the Kubernetes node object.
+  replicaZoneSoftAntiAffinity: ~
+  # -- Setting that allows scheduling on disks with existing healthy replicas of the same volume. This setting is enabled by default.
+  replicaDiskSoftAntiAffinity: ~
+  # -- Policy that defines the action Longhorn takes when a volume is stuck with a StatefulSet or Deployment pod on a node that failed.
+  nodeDownPodDeletionPolicy: ~
+  # -- Policy that defines the action Longhorn takes when a node with the last healthy replica of a volume is drained.
+  nodeDrainPolicy: ~
+  # -- Setting that allows automatic detaching of manually-attached volumes when a node is cordoned.
+  detachManuallyAttachedVolumesWhenCordoned: ~
+  # -- Number of seconds that Longhorn waits before reusing existing data on a failed replica instead of creating a new replica of a degraded volume.
+  replicaReplenishmentWaitInterval: ~
+  # -- Maximum number of replicas that can be concurrently rebuilt on each node.
+  concurrentReplicaRebuildPerNodeLimit: ~
+  # -- Maximum number of volumes that can be concurrently restored on each node using a backup. When the value is "0", restoration of volumes using a backup is disabled.
+  concurrentVolumeBackupRestorePerNodeLimit: ~
+  # -- Setting that disables the revision counter and thereby prevents Longhorn from tracking all write operations to a volume. When salvaging a volume, Longhorn uses properties of the "volume-head-xxx.img" file (the last file size and the last time the file was modified) to select the replica to be used for volume recovery. This setting applies only to volumes created using the Longhorn UI.
+  disableRevisionCounter: "true"
+  # -- Image pull policy for system-managed pods, such as Instance Manager, engine images, and CSI Driver. Changes to the image pull policy are applied only after the system-managed pods restart.
+  systemManagedPodsImagePullPolicy: ~
+  # -- Setting that allows you to create and attach a volume without having all replicas scheduled at the time of creation.
+  allowVolumeCreationWithDegradedAvailability: ~
+  # -- Setting that allows Longhorn to automatically clean up the system-generated snapshot after replica rebuilding is completed.
+  autoCleanupSystemGeneratedSnapshot: ~
+  # -- Setting that allows Longhorn to automatically clean up the snapshot generated by a recurring backup job.
+  autoCleanupRecurringJobBackupSnapshot: ~
+  # -- Maximum number of engines that are allowed to concurrently upgrade on each node after Longhorn Manager is upgraded. When the value is "0", Longhorn does not automatically upgrade volume engines to the new default engine image version.
+  concurrentAutomaticEngineUpgradePerNodeLimit: ~
+  # -- Number of minutes that Longhorn waits before cleaning up the backing image file when no replicas in the disk are using it.
+  backingImageCleanupWaitInterval: ~
+  # -- Number of seconds that Longhorn waits before downloading a backing image file again when the status of all image disk files changes to "failed" or "unknown".
+  backingImageRecoveryWaitInterval: ~
+  # -- Percentage of the total allocatable CPU resources on each node to be reserved for each instance manager pod when the V1 Data Engine is enabled. The default value is "12".
+  guaranteedInstanceManagerCPU: ~
+  # -- Setting that notifies Longhorn that the cluster is using the Kubernetes Cluster Autoscaler.
+  kubernetesClusterAutoscalerEnabled: ~
+  # -- Setting that allows Longhorn to automatically delete an orphaned resource and the corresponding data (for example, stale replicas). Orphaned resources on failed or unknown nodes are not automatically cleaned up.
+  orphanAutoDeletion: ~
+  # -- Storage network for in-cluster traffic. When unspecified, Longhorn uses the Kubernetes cluster network.
+  storageNetwork: ~
+  # -- Flag that prevents accidental uninstallation of Longhorn.
+  deletingConfirmationFlag: ~
+  # -- Timeout between the Longhorn Engine and replicas. Specify a value between "8" and "30" seconds. The default value is "8".
+  engineReplicaTimeout: ~
+  # -- Setting that allows you to enable and disable snapshot hashing and data integrity checks.
+  snapshotDataIntegrity: ~
+  # -- Setting that allows disabling of snapshot hashing after snapshot creation to minimize impact on system performance.
+  snapshotDataIntegrityImmediateCheckAfterSnapshotCreation: ~
+  # -- Setting that defines when Longhorn checks the integrity of data in snapshot disk files. You must use the Unix cron expression format.
+  snapshotDataIntegrityCronjob: ~
+  # -- Setting that allows Longhorn to automatically mark the latest snapshot and its parent files as removed during a filesystem trim. Longhorn does not remove snapshots containing multiple child files.
+  removeSnapshotsDuringFilesystemTrim: ~
+  # -- Setting that allows fast rebuilding of replicas using the checksum of snapshot disk files. Before enabling this setting, you must set the snapshot-data-integrity value to "enable" or "fast-check".
+  fastReplicaRebuildEnabled: ~
+  # -- Number of seconds that an HTTP client waits for a response from a File Sync server before considering the connection to have failed.
+  replicaFileSyncHttpClientTimeout: ~
+  # -- Number of seconds that Longhorn allows for the completion of replica rebuilding and snapshot cloning operations.
+  longGRPCTimeOut: ~
+  # -- Log levels that indicate the type and severity of logs in Longhorn Manager. The default value is "Info". (Options: "Panic", "Fatal", "Error", "Warn", "Info", "Debug", "Trace")
+  logLevel: ~
+  # -- Setting that allows you to specify a backup compression method.
+  backupCompressionMethod: ~
+  # -- Maximum number of worker threads that can concurrently run for each backup.
+  backupConcurrentLimit: ~
+  # -- Maximum number of worker threads that can concurrently run for each restore operation.
+  restoreConcurrentLimit: ~
+  # -- Setting that allows you to enable the V1 Data Engine.
+  v1DataEngine: ~
+  # -- Setting that allows you to enable the V2 Data Engine, which is based on the Storage Performance Development Kit (SPDK). The V2 Data Engine is an experimental feature and should not be used in production environments.
+  v2DataEngine: ~
+  # -- Setting that allows you to configure maximum huge page size (in MiB) for the V2 Data Engine.
+  v2DataEngineHugepageLimit: ~
+  # -- Number of millicpus on each node to be reserved for each Instance Manager pod when the V2 Data Engine is enabled. The default value is "1250".
+  v2DataEngineGuaranteedInstanceManagerCPU: ~
+  # -- CPU cores on which the Storage Performance Development Kit (SPDK) target daemon should run. The SPDK target daemon is located in each Instance Manager pod. Ensure that the number of cores is less than or equal to the guaranteed Instance Manager CPUs for the V2 Data Engine. The default value is "0x1".
+  v2DataEngineCPUMask: ~
+  # -- Setting that allows scheduling of empty node selector volumes to any node.
+  allowEmptyNodeSelectorVolume: ~
+  # -- Setting that allows scheduling of empty disk selector volumes to any disk.
+  allowEmptyDiskSelectorVolume: ~
+  # -- Setting that allows Longhorn to periodically collect anonymous usage data for product improvement purposes. Longhorn sends collected data to the [Upgrade Responder](https://github.com/longhorn/upgrade-responder) server, which is the data source of the Longhorn Public Metrics Dashboard (https://metrics.longhorn.io). The Upgrade Responder server does not store data that can be used to identify clients, including IP addresses.
+  allowCollectingLonghornUsageMetrics: ~
+  # -- Setting that temporarily prevents all attempts to purge volume snapshots.
+  disableSnapshotPurge: ~
+  # -- Maximum snapshot count for a volume. The value should be between 2 to 250
+  snapshotMaxCount: ~
+  # -- Setting that allows you to configure the log level of the SPDK target daemon (spdk_tgt) of the V2 Data Engine.
+  v2DataEngineLogLevel: ~
+  # -- Setting that allows you to configure the log flags of the SPDK target daemon (spdk_tgt) of the V2 Data Engine.
+  v2DataEngineLogFlags: ~
+  # -- Setting that freezes the filesystem on the root partition before a snapshot is created.
+  freezeFilesystemForSnapshot: ~
+  # -- Setting that automatically cleans up the snapshot when the backup is deleted.
+  autoCleanupSnapshotWhenDeleteBackup: ~
+  # -- Setting that allows Longhorn to detect node failure and immediately migrate affected RWX volumes.
+  rwxVolumeFastFailover: ~
+
+# -- Setting that allows you to update the default backupstore.
+defaultBackupStore:
+  # -- Endpoint used to access the default backupstore. (Options: "NFS", "CIFS", "AWS", "GCP", "AZURE")
+  backupTarget: s3://backupbucket@us-east-1/
+  # -- Name of the Kubernetes secret associated with the default backup target.
+  backupTargetCredentialSecret: minio-secret
+  # -- Number of seconds that Longhorn waits before checking the default backupstore for new backups. The default value is "300". When the value is "0", polling is disabled.
+  pollInterval: ~
+
+privateRegistry:
+  # -- Setting that allows you to create a private registry secret.
+  createSecret: ~
+  # -- URL of a private registry. When unspecified, Longhorn uses the default system registry.
+  registryUrl: ~
+  # -- User account used for authenticating with a private registry.
+  registryUser: ~
+  # -- Password for authenticating with a private registry.
+  registryPasswd: ~
+  # -- Kubernetes secret that allows you to pull images from a private registry. This setting applies only when creation of private registry secrets is enabled. You must include the private registry name in the secret name.
+  registrySecret: ~
+
+longhornManager:
+  log:
+    # -- Format of Longhorn Manager logs. (Options: "plain", "json")
+    format: plain
+  # -- PriorityClass for Longhorn Manager.
+  priorityClass: *defaultPriorityClassNameRef
+  # -- Toleration for Longhorn Manager on nodes allowed to run Longhorn components.
+  tolerations: []
+  ## If you want to set tolerations for Longhorn Manager DaemonSet, delete the `[]` in the line above
+  ## and uncomment this example block
+  # - key: "key"
+  #   operator: "Equal"
+  #   value: "value"
+  #   effect: "NoSchedule"
+  # -- Node selector for Longhorn Manager. Specify the nodes allowed to run Longhorn Manager.
+  nodeSelector: {}
+  ## If you want to set node selector for Longhorn Manager DaemonSet, delete the `{}` in the line above
+  ## and uncomment this example block
+  #  label-key1: "label-value1"
+  #  label-key2: "label-value2"
+  # -- Annotation for the Longhorn Manager service.
+  serviceAnnotations: {}
+  ## If you want to set annotations for the Longhorn Manager service, delete the `{}` in the line above
+  ## and uncomment this example block
+  #  annotation-key1: "annotation-value1"
+  #  annotation-key2: "annotation-value2"
+
+longhornDriver:
+  log:
+    # -- Format of longhorn-driver logs. (Options: "plain", "json")
+    format: plain
+  # -- PriorityClass for Longhorn Driver.
+  priorityClass: *defaultPriorityClassNameRef
+  # -- Toleration for Longhorn Driver on nodes allowed to run Longhorn components.
+  tolerations: []
+  ## If you want to set tolerations for Longhorn Driver Deployer Deployment, delete the `[]` in the line above
+  ## and uncomment this example block
+  # - key: "key"
+  #   operator: "Equal"
+  #   value: "value"
+  #   effect: "NoSchedule"
+  # -- Node selector for Longhorn Driver. Specify the nodes allowed to run Longhorn Driver.
+  nodeSelector: {}
+  ## If you want to set node selector for Longhorn Driver Deployer Deployment, delete the `{}` in the line above
+  ## and uncomment this example block
+  #  label-key1: "label-value1"
+  #  label-key2: "label-value2"
+
+longhornUI:
+  # -- Replica count for Longhorn UI.
+  replicas: 1
+  # -- PriorityClass for Longhorn UI.
+  priorityClass: *defaultPriorityClassNameRef
+  # -- Toleration for Longhorn UI on nodes allowed to run Longhorn components.
+  tolerations: []
+  ## If you want to set tolerations for Longhorn UI Deployment, delete the `[]` in the line above
+  ## and uncomment this example block
+  # - key: "key"
+  #   operator: "Equal"
+  #   value: "value"
+  #   effect: "NoSchedule"
+  # -- Node selector for Longhorn UI. Specify the nodes allowed to run Longhorn UI.
+  nodeSelector: {}
+  ## If you want to set node selector for Longhorn UI Deployment, delete the `{}` in the line above
+  ## and uncomment this example block
+  #  label-key1: "label-value1"
+  #  label-key2: "label-value2"
+
+ingress:
+  # -- Setting that allows Longhorn to generate ingress records for the Longhorn UI service.
+  enabled: false
+
+  # -- IngressClass resource that contains ingress configuration, including the name of the Ingress controller.
+  # ingressClassName can replace the kubernetes.io/ingress.class annotation used in earlier Kubernetes releases.
+  ingressClassName: ~
+
+  # -- Hostname of the Layer 7 load balancer.
+  host: sslip.io
+
+  # -- Setting that allows you to enable TLS on ingress records.
+  tls: false
+
+  # -- Setting that allows you to enable secure connections to the Longhorn UI service via port 443.
+  secureBackends: false
+
+  # -- TLS secret that contains the private key and certificate to be used for TLS. This setting applies only when TLS is enabled on ingress records.
+  tlsSecret: longhorn.local-tls
+
+  # -- Default ingress path. You can access the Longhorn UI by following the full ingress path {{host}}+{{path}}.
+  path: /
+
+  # -- Ingress path type. To maintain backward compatibility, the default value is "ImplementationSpecific".
+  pathType: ImplementationSpecific
+
+  ## If you're using kube-lego, you will want to add:
+  ## kubernetes.io/tls-acme: true
+  ##
+  ## For a full list of possible ingress annotations, please see
+  ## ref: https://github.com/kubernetes/ingress-nginx/blob/master/docs/annotations.md
+  ##
+  ## If tls is set to true, annotation ingress.kubernetes.io/secure-backends: "true" will automatically be set
+  # -- Ingress annotations in the form of key-value pairs.
+  annotations:
+  #  kubernetes.io/ingress.class: nginx
+  #  kubernetes.io/tls-acme: true
+
+  # -- Secret that contains a TLS private key and certificate. Use secrets if you want to use your own certificates to secure ingresses.
+  secrets:
+  ## If you're providing your own certificates, please use this to add the certificates as secrets
+  ## key and certificate should start with -----BEGIN CERTIFICATE----- or
+  ## -----BEGIN RSA PRIVATE KEY-----
+  ##
+  ## name should line up with a tlsSecret set further up
+  ## If you're using kube-lego, this is unneeded, as it will create the secret for you if it is not set
+  ##
+  ## It is also possible to create and manage the certificates outside of this helm chart
+  ## Please see README.md for more information
+  # - name: longhorn.local-tls
+  #   key:
+  #   certificate:
+
+# -- Setting that allows you to enable pod security policies (PSPs) that allow privileged Longhorn pods to start. This setting applies only to clusters running Kubernetes 1.25 and earlier, and with the built-in Pod Security admission controller enabled.
+enablePSP: false
+
+# -- Specify override namespace, specifically this is useful for using longhorn as sub-chart and its release namespace is not the `longhorn-system`.
+namespaceOverride: ""
+
+# -- Annotation for the Longhorn Manager DaemonSet pods. This setting is optional.
+annotations: {}
+
+serviceAccount:
+  # -- Annotations to add to the service account
+  annotations: {}
+
+metrics:
+  serviceMonitor:
+    # -- Setting that allows the creation of a Prometheus ServiceMonitor resource for Longhorn Manager components.
+    enabled: false
+    # -- Additional labels for the Prometheus ServiceMonitor resource.
+    additionalLabels: {}
+    # -- Annotations for the Prometheus ServiceMonitor resource.
+    annotations: {}
+    # -- Interval at which Prometheus scrapes the metrics from the target.
+    interval: ""
+    # -- Timeout after which Prometheus considers the scrape to be failed.
+    scrapeTimeout: ""
+    # -- Configures the relabeling rules to apply the target’s metadata labels. See the [Prometheus Operator
+    # documentation](https://prometheus-operator.dev/docs/api-reference/api/#monitoring.coreos.com/v1.Endpoint) for
+    # formatting details.
+    relabelings: []
+    # -- Configures the relabeling rules to apply to the samples before ingestion. See the [Prometheus Operator
+    # documentation](https://prometheus-operator.dev/docs/api-reference/api/#monitoring.coreos.com/v1.Endpoint) for
+    # formatting details.
+    metricRelabelings: []
+
+## openshift settings
+openshift:
+  # -- Setting that allows Longhorn to integrate with OpenShift.
+  enabled: false
+  ui:
+    # -- Route for connections between Longhorn and the OpenShift web console.
+    route: "longhorn-ui"
+    # -- Port for accessing the OpenShift web console.
+    port: 443
+    # -- Port for proxy that provides access to the OpenShift web console.
+    proxy: 8443
+
+# -- Setting that allows Longhorn to generate code coverage profiles.
+enableGoCoverDir: false
diff --git a/longhorn-system/longhorn.yaml b/longhorn-system/longhorn.yaml
new file mode 100644
index 0000000..6ea349c
--- /dev/null
+++ b/longhorn-system/longhorn.yaml
@@ -0,0 +1,5421 @@
+---
+# Source: longhorn/templates/priorityclass.yaml
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: "longhorn-critical"
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+description: "Ensure Longhorn pods have the highest priority to prevent any unexpected eviction by the Kubernetes scheduler under node pressure"
+globalDefault: false
+preemptionPolicy: PreemptLowerPriority
+value: 1000000000
+---
+# Source: longhorn/templates/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: longhorn-service-account
+  namespace: longhorn-system
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+---
+# Source: longhorn/templates/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: longhorn-ui-service-account
+  namespace: longhorn-system
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+---
+# Source: longhorn/templates/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: longhorn-support-bundle
+  namespace: longhorn-system
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+---
+# Source: longhorn/templates/default-resource.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: longhorn-default-resource
+  namespace: longhorn-system
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+data:
+  default-resource.yaml: |-
+    backup-target: s3://backupbucket@us-east-1/
+    backup-target-credential-secret: minio-secret
+---
+# Source: longhorn/templates/default-setting.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: longhorn-default-setting
+  namespace: longhorn-system
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+data:
+  default-setting.yaml: |-
+    default-data-path: /var/lib/longhorn/
+    storage-minimal-available-percentage: 10
+    storage-reserved-percentage-for-default-disk: 10
+    default-replica-count: 1
+    default-data-locality: best-effort
+    priority-class: longhorn-critical
+    disable-revision-counter: true
+---
+# Source: longhorn/templates/storageclass.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: longhorn-storageclass
+  namespace: longhorn-system
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+data:
+  storageclass.yaml: |
+    kind: StorageClass
+    apiVersion: storage.k8s.io/v1
+    metadata:
+      name: longhorn
+      annotations:
+        storageclass.kubernetes.io/is-default-class: "true"
+    provisioner: driver.longhorn.io
+    allowVolumeExpansion: true
+    reclaimPolicy: "Delete"
+    volumeBindingMode: Immediate
+    parameters:
+      numberOfReplicas: "1"
+      staleReplicaTimeout: "30"
+      fromBackup: ""
+      fsType: "ext4"
+      dataLocality: "best-effort"
+      unmapMarkSnapChainRemoved: "ignored"
+      disableRevisionCounter: "true"
+      dataEngine: "v1"
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: backingimagedatasources.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: BackingImageDataSource
+    listKind: BackingImageDataSourceList
+    plural: backingimagedatasources
+    shortNames:
+    - lhbids
+    singular: backingimagedatasource
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The current state of the pod used to provision the backing image
+        file from source
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The data source type
+      jsonPath: .spec.sourceType
+      name: SourceType
+      type: string
+    - description: The node the backing image file will be prepared on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - description: The disk the backing image file will be prepared on
+      jsonPath: .spec.diskUUID
+      name: DiskUUID
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: BackingImageDataSource is where Longhorn stores backing image
+          data source object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The system generated UUID of the provisioned backing image file
+      jsonPath: .spec.uuid
+      name: UUID
+      type: string
+    - description: The current state of the pod used to provision the backing image
+        file from source
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The data source type
+      jsonPath: .spec.sourceType
+      name: SourceType
+      type: string
+    - description: The backing image file size
+      jsonPath: .status.size
+      name: Size
+      type: string
+    - description: The node the backing image file will be prepared on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - description: The disk the backing image file will be prepared on
+      jsonPath: .spec.diskUUID
+      name: DiskUUID
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: BackingImageDataSource is where Longhorn stores backing image
+          data source object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: BackingImageDataSourceSpec defines the desired state of the
+              Longhorn backing image data source
+            properties:
+              checksum:
+                type: string
+              diskPath:
+                type: string
+              diskUUID:
+                type: string
+              fileTransferred:
+                type: boolean
+              nodeID:
+                type: string
+              parameters:
+                additionalProperties:
+                  type: string
+                type: object
+              sourceType:
+                enum:
+                - download
+                - upload
+                - export-from-volume
+                - restore
+                - clone
+                type: string
+              uuid:
+                type: string
+            type: object
+          status:
+            description: BackingImageDataSourceStatus defines the observed state of
+              the Longhorn backing image data source
+            properties:
+              checksum:
+                type: string
+              currentState:
+                type: string
+              ip:
+                type: string
+              message:
+                type: string
+              ownerID:
+                type: string
+              progress:
+                type: integer
+              runningParameters:
+                additionalProperties:
+                  type: string
+                nullable: true
+                type: object
+              size:
+                format: int64
+                type: integer
+              storageIP:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: backingimagemanagers.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: BackingImageManager
+    listKind: BackingImageManagerList
+    plural: backingimagemanagers
+    shortNames:
+    - lhbim
+    singular: backingimagemanager
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The current state of the manager
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The image the manager pod will use
+      jsonPath: .spec.image
+      name: Image
+      type: string
+    - description: The node the manager is on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - description: The disk the manager is responsible for
+      jsonPath: .spec.diskUUID
+      name: DiskUUID
+      type: string
+    - description: The disk path the manager is using
+      jsonPath: .spec.diskPath
+      name: DiskPath
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: BackingImageManager is where Longhorn stores backing image manager
+          object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The current state of the manager
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The image the manager pod will use
+      jsonPath: .spec.image
+      name: Image
+      type: string
+    - description: The node the manager is on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - description: The disk the manager is responsible for
+      jsonPath: .spec.diskUUID
+      name: DiskUUID
+      type: string
+    - description: The disk path the manager is using
+      jsonPath: .spec.diskPath
+      name: DiskPath
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: BackingImageManager is where Longhorn stores backing image manager
+          object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: BackingImageManagerSpec defines the desired state of the
+              Longhorn backing image manager
+            properties:
+              backingImages:
+                additionalProperties:
+                  type: string
+                type: object
+              diskPath:
+                type: string
+              diskUUID:
+                type: string
+              image:
+                type: string
+              nodeID:
+                type: string
+            type: object
+          status:
+            description: BackingImageManagerStatus defines the observed state of the
+              Longhorn backing image manager
+            properties:
+              apiMinVersion:
+                type: integer
+              apiVersion:
+                type: integer
+              backingImageFileMap:
+                additionalProperties:
+                  properties:
+                    currentChecksum:
+                      type: string
+                    message:
+                      type: string
+                    name:
+                      type: string
+                    progress:
+                      type: integer
+                    realSize:
+                      format: int64
+                      type: integer
+                    senderManagerAddress:
+                      type: string
+                    sendingReference:
+                      type: integer
+                    size:
+                      format: int64
+                      type: integer
+                    state:
+                      type: string
+                    uuid:
+                      type: string
+                    virtualSize:
+                      format: int64
+                      type: integer
+                  type: object
+                nullable: true
+                type: object
+              currentState:
+                type: string
+              ip:
+                type: string
+              ownerID:
+                type: string
+              storageIP:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: backingimages.longhorn.io
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          name: longhorn-conversion-webhook
+          namespace: longhorn-system
+          path: /v1/webhook/conversion
+          port: 9501
+      conversionReviewVersions:
+      - v1beta2
+      - v1beta1
+  group: longhorn.io
+  names:
+    kind: BackingImage
+    listKind: BackingImageList
+    plural: backingimages
+    shortNames:
+    - lhbi
+    singular: backingimage
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The backing image name
+      jsonPath: .spec.image
+      name: Image
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: BackingImage is where Longhorn stores backing image object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The system generated UUID
+      jsonPath: .status.uuid
+      name: UUID
+      type: string
+    - description: The source of the backing image file data
+      jsonPath: .spec.sourceType
+      name: SourceType
+      type: string
+    - description: The backing image file size in each disk
+      jsonPath: .status.size
+      name: Size
+      type: string
+    - description: The virtual size of the image (may be larger than file size)
+      jsonPath: .status.virtualSize
+      name: VirtualSize
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: BackingImage is where Longhorn stores backing image object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: BackingImageSpec defines the desired state of the Longhorn
+              backing image
+            properties:
+              checksum:
+                type: string
+              dataEngine:
+                default: v1
+                enum:
+                - v1
+                - v2
+                type: string
+              diskFileSpecMap:
+                additionalProperties:
+                  properties:
+                    dataEngine:
+                      enum:
+                      - v1
+                      - v2
+                      type: string
+                    evictionRequested:
+                      type: boolean
+                  type: object
+                type: object
+              diskSelector:
+                items:
+                  type: string
+                type: array
+              disks:
+                additionalProperties:
+                  type: string
+                description: Deprecated. We are now using DiskFileSpecMap to assign
+                  different spec to the file on different disks.
+                type: object
+              minNumberOfCopies:
+                type: integer
+              nodeSelector:
+                items:
+                  type: string
+                type: array
+              secret:
+                type: string
+              secretNamespace:
+                type: string
+              sourceParameters:
+                additionalProperties:
+                  type: string
+                type: object
+              sourceType:
+                enum:
+                - download
+                - upload
+                - export-from-volume
+                - restore
+                - clone
+                type: string
+            type: object
+          status:
+            description: BackingImageStatus defines the observed state of the Longhorn
+              backing image status
+            properties:
+              checksum:
+                type: string
+              diskFileStatusMap:
+                additionalProperties:
+                  properties:
+                    dataEngine:
+                      enum:
+                      - v1
+                      - v2
+                      type: string
+                    lastStateTransitionTime:
+                      type: string
+                    message:
+                      type: string
+                    progress:
+                      type: integer
+                    state:
+                      type: string
+                  type: object
+                nullable: true
+                type: object
+              diskLastRefAtMap:
+                additionalProperties:
+                  type: string
+                nullable: true
+                type: object
+              ownerID:
+                type: string
+              realSize:
+                description: Real size of image in bytes, which may be smaller than
+                  the size when the file is a sparse file. Will be zero until known
+                  (e.g. while a backing image is uploading)
+                format: int64
+                type: integer
+              size:
+                format: int64
+                type: integer
+              uuid:
+                type: string
+              v2FirstCopyDisk:
+                type: string
+              v2FirstCopyStatus:
+                description: It is pending -> in-progress -> ready/failed
+                type: string
+              virtualSize:
+                description: Virtual size of image in bytes, which may be larger than
+                  physical size. Will be zero until known (e.g. while a backing image
+                  is uploading)
+                format: int64
+                type: integer
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: backupbackingimages.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: BackupBackingImage
+    listKind: BackupBackingImageList
+    plural: backupbackingimages
+    shortNames:
+    - lhbbi
+    singular: backupbackingimage
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The backing image name
+      jsonPath: .status.backingImage
+      name: BackingImage
+      type: string
+    - description: The backing image size
+      jsonPath: .status.size
+      name: Size
+      type: string
+    - description: The backing image backup upload finished time
+      jsonPath: .status.backupCreatedAt
+      name: BackupCreatedAt
+      type: string
+    - description: The backing image backup state
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The last synced time
+      jsonPath: .status.lastSyncedAt
+      name: LastSyncedAt
+      type: string
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: BackupBackingImage is where Longhorn stores backing image backup
+          object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: BackupBackingImageSpec defines the desired state of the Longhorn
+              backing image backup
+            properties:
+              backingImage:
+                description: |-
+                  The backing image name.
+                type: string
+              backupTargetName:
+                description: The backup target name.
+                nullable: true
+                type: string
+              labels:
+                additionalProperties:
+                  type: string
+                description: The labels of backing image backup.
+                type: object
+              syncRequestedAt:
+                description: The time to request run sync the remote backing image
+                  backup.
+                format: date-time
+                nullable: true
+                type: string
+              userCreated:
+                description: |-
+                  Is this CR created by user through API or UI.
+                type: boolean
+            required:
+            - backingImage
+            - userCreated
+            type: object
+          status:
+            description: BackupBackingImageStatus defines the observed state of the
+              Longhorn backing image backup
+            properties:
+              backingImage:
+                description: The backing image name.
+                type: string
+              backupCreatedAt:
+                description: The backing image backup upload finished time.
+                type: string
+              checksum:
+                description: The checksum of the backing image.
+                type: string
+              compressionMethod:
+                description: Compression method
+                type: string
+              error:
+                description: The error message when taking the backing image backup.
+                type: string
+              labels:
+                additionalProperties:
+                  type: string
+                description: The labels of backing image backup.
+                nullable: true
+                type: object
+              lastSyncedAt:
+                description: The last time that the backing image backup was synced
+                  with the remote backup target.
+                format: date-time
+                nullable: true
+                type: string
+              managerAddress:
+                description: The address of the backing image manager that runs backing
+                  image backup.
+                type: string
+              messages:
+                additionalProperties:
+                  type: string
+                description: The error messages when listing or inspecting backing
+                  image backup.
+                nullable: true
+                type: object
+              ownerID:
+                description: The node ID on which the controller is responsible to
+                  reconcile this CR.
+                type: string
+              progress:
+                description: The backing image backup progress.
+                type: integer
+              secret:
+                description: Record the secret if this backup backing image is encrypted
+                type: string
+              secretNamespace:
+                description: Record the secret namespace if this backup backing image
+                  is encrypted
+                type: string
+              size:
+                description: The backing image size.
+                format: int64
+                type: integer
+              state:
+                description: |-
+                  The backing image backup creation state.
+                  Can be "", "InProgress", "Completed", "Error", "Unknown".
+                type: string
+              url:
+                description: The backing image backup URL.
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: backups.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: Backup
+    listKind: BackupList
+    plural: backups
+    shortNames:
+    - lhb
+    singular: backup
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The snapshot name
+      jsonPath: .status.snapshotName
+      name: SnapshotName
+      type: string
+    - description: The snapshot size
+      jsonPath: .status.size
+      name: SnapshotSize
+      type: string
+    - description: The snapshot creation time
+      jsonPath: .status.snapshotCreatedAt
+      name: SnapshotCreatedAt
+      type: string
+    - description: The backup state
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The backup last synced time
+      jsonPath: .status.lastSyncedAt
+      name: LastSyncedAt
+      type: string
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: Backup is where Longhorn stores backup object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The snapshot name
+      jsonPath: .status.snapshotName
+      name: SnapshotName
+      type: string
+    - description: The snapshot size
+      jsonPath: .status.size
+      name: SnapshotSize
+      type: string
+    - description: The snapshot creation time
+      jsonPath: .status.snapshotCreatedAt
+      name: SnapshotCreatedAt
+      type: string
+    - description: The backup target name
+      jsonPath: .status.backupTargetName
+      name: BackupTarget
+      type: string
+    - description: The backup state
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The backup last synced time
+      jsonPath: .status.lastSyncedAt
+      name: LastSyncedAt
+      type: string
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: Backup is where Longhorn stores backup object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: BackupSpec defines the desired state of the Longhorn backup
+            properties:
+              backupMode:
+                description: |-
+                  The backup mode of this backup.
+                  Can be "full" or "incremental"
+                enum:
+                - full
+                - incremental
+                - ""
+                type: string
+              labels:
+                additionalProperties:
+                  type: string
+                description: The labels of snapshot backup.
+                type: object
+              snapshotName:
+                description: The snapshot name.
+                type: string
+              syncRequestedAt:
+                description: The time to request run sync the remote backup.
+                format: date-time
+                nullable: true
+                type: string
+            type: object
+          status:
+            description: BackupStatus defines the observed state of the Longhorn backup
+            properties:
+              backupCreatedAt:
+                description: The snapshot backup upload finished time.
+                type: string
+              backupTargetName:
+                description: The backup target name.
+                type: string
+              compressionMethod:
+                description: Compression method
+                type: string
+              error:
+                description: The error message when taking the snapshot backup.
+                type: string
+              labels:
+                additionalProperties:
+                  type: string
+                description: The labels of snapshot backup.
+                nullable: true
+                type: object
+              lastSyncedAt:
+                description: The last time that the backup was synced with the remote
+                  backup target.
+                format: date-time
+                nullable: true
+                type: string
+              messages:
+                additionalProperties:
+                  type: string
+                description: The error messages when calling longhorn engine on listing
+                  or inspecting backups.
+                nullable: true
+                type: object
+              newlyUploadDataSize:
+                description: Size in bytes of newly uploaded data
+                type: string
+              ownerID:
+                description: The node ID on which the controller is responsible to
+                  reconcile this backup CR.
+                type: string
+              progress:
+                description: The snapshot backup progress.
+                type: integer
+              reUploadedDataSize:
+                description: Size in bytes of reuploaded data
+                type: string
+              replicaAddress:
+                description: The address of the replica that runs snapshot backup.
+                type: string
+              size:
+                description: The snapshot size.
+                type: string
+              snapshotCreatedAt:
+                description: The snapshot creation time.
+                type: string
+              snapshotName:
+                description: The snapshot name.
+                type: string
+              state:
+                description: |-
+                  The backup creation state.
+                  Can be "", "InProgress", "Completed", "Error", "Unknown".
+                type: string
+              url:
+                description: The snapshot backup URL.
+                type: string
+              volumeBackingImageName:
+                description: The volume's backing image name.
+                type: string
+              volumeCreated:
+                description: The volume creation time.
+                type: string
+              volumeName:
+                description: The volume name.
+                type: string
+              volumeSize:
+                description: The volume size.
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: backuptargets.longhorn.io
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          name: longhorn-conversion-webhook
+          namespace: longhorn-system
+          path: /v1/webhook/conversion
+          port: 9501
+      conversionReviewVersions:
+      - v1beta2
+      - v1beta1
+  group: longhorn.io
+  names:
+    kind: BackupTarget
+    listKind: BackupTargetList
+    plural: backuptargets
+    shortNames:
+    - lhbt
+    singular: backuptarget
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The backup target URL
+      jsonPath: .spec.backupTargetURL
+      name: URL
+      type: string
+    - description: The backup target credential secret
+      jsonPath: .spec.credentialSecret
+      name: Credential
+      type: string
+    - description: The backup target poll interval
+      jsonPath: .spec.pollInterval
+      name: LastBackupAt
+      type: string
+    - description: Indicate whether the backup target is available or not
+      jsonPath: .status.available
+      name: Available
+      type: boolean
+    - description: The backup target last synced time
+      jsonPath: .status.lastSyncedAt
+      name: LastSyncedAt
+      type: string
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: BackupTarget is where Longhorn stores backup target object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The backup target URL
+      jsonPath: .spec.backupTargetURL
+      name: URL
+      type: string
+    - description: The backup target credential secret
+      jsonPath: .spec.credentialSecret
+      name: Credential
+      type: string
+    - description: The backup target poll interval
+      jsonPath: .spec.pollInterval
+      name: LastBackupAt
+      type: string
+    - description: Indicate whether the backup target is available or not
+      jsonPath: .status.available
+      name: Available
+      type: boolean
+    - description: The backup target last synced time
+      jsonPath: .status.lastSyncedAt
+      name: LastSyncedAt
+      type: string
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: BackupTarget is where Longhorn stores backup target object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: BackupTargetSpec defines the desired state of the Longhorn
+              backup target
+            properties:
+              backupTargetURL:
+                description: The backup target URL.
+                type: string
+              credentialSecret:
+                description: The backup target credential secret.
+                type: string
+              pollInterval:
+                description: The interval that the cluster needs to run sync with
+                  the backup target.
+                type: string
+              syncRequestedAt:
+                description: The time to request run sync the remote backup target.
+                format: date-time
+                nullable: true
+                type: string
+            type: object
+          status:
+            description: BackupTargetStatus defines the observed state of the Longhorn
+              backup target
+            properties:
+              available:
+                description: Available indicates if the remote backup target is available
+                  or not.
+                type: boolean
+              conditions:
+                description: Records the reason on why the backup target is unavailable.
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status
+                        to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about
+                        last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's
+                        last transition.
+                      type: string
+                    status:
+                      description: |-
+                        Status is the status of the condition.
+                        Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                nullable: true
+                type: array
+              lastSyncedAt:
+                description: The last time that the controller synced with the remote
+                  backup target.
+                format: date-time
+                nullable: true
+                type: string
+              ownerID:
+                description: The node ID on which the controller is responsible to
+                  reconcile this backup target CR.
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: backupvolumes.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: BackupVolume
+    listKind: BackupVolumeList
+    plural: backupvolumes
+    shortNames:
+    - lhbv
+    singular: backupvolume
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The backup volume creation time
+      jsonPath: .status.createdAt
+      name: CreatedAt
+      type: string
+    - description: The backup volume last backup name
+      jsonPath: .status.lastBackupName
+      name: LastBackupName
+      type: string
+    - description: The backup volume last backup time
+      jsonPath: .status.lastBackupAt
+      name: LastBackupAt
+      type: string
+    - description: The backup volume last synced time
+      jsonPath: .status.lastSyncedAt
+      name: LastSyncedAt
+      type: string
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: BackupVolume is where Longhorn stores backup volume object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The backup target name
+      jsonPath: .spec.backupTargetName
+      name: BackupTarget
+      type: string
+    - description: The backup volume creation time
+      jsonPath: .status.createdAt
+      name: CreatedAt
+      type: string
+    - description: The backup volume last backup name
+      jsonPath: .status.lastBackupName
+      name: LastBackupName
+      type: string
+    - description: The backup volume last backup time
+      jsonPath: .status.lastBackupAt
+      name: LastBackupAt
+      type: string
+    - description: The backup volume last synced time
+      jsonPath: .status.lastSyncedAt
+      name: LastSyncedAt
+      type: string
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: BackupVolume is where Longhorn stores backup volume object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: BackupVolumeSpec defines the desired state of the Longhorn
+              backup volume
+            properties:
+              backupTargetName:
+                description: The backup target name that the backup volume was synced.
+                nullable: true
+                type: string
+              syncRequestedAt:
+                description: The time to request run sync the remote backup volume.
+                format: date-time
+                nullable: true
+                type: string
+              volumeName:
+                description: The volume name that the backup volume was used to backup.
+                type: string
+            type: object
+          status:
+            description: BackupVolumeStatus defines the observed state of the Longhorn
+              backup volume
+            properties:
+              backingImageChecksum:
+                description: the backing image checksum.
+                type: string
+              backingImageName:
+                description: The backing image name.
+                type: string
+              createdAt:
+                description: The backup volume creation time.
+                type: string
+              dataStored:
+                description: The backup volume block count.
+                type: string
+              labels:
+                additionalProperties:
+                  type: string
+                description: The backup volume labels.
+                nullable: true
+                type: object
+              lastBackupAt:
+                description: The latest volume backup time.
+                type: string
+              lastBackupName:
+                description: The latest volume backup name.
+                type: string
+              lastModificationTime:
+                description: The backup volume config last modification time.
+                format: date-time
+                nullable: true
+                type: string
+              lastSyncedAt:
+                description: The last time that the backup volume was synced into
+                  the cluster.
+                format: date-time
+                nullable: true
+                type: string
+              messages:
+                additionalProperties:
+                  type: string
+                description: The error messages when call longhorn engine on list
+                  or inspect backup volumes.
+                nullable: true
+                type: object
+              ownerID:
+                description: The node ID on which the controller is responsible to
+                  reconcile this backup volume CR.
+                type: string
+              size:
+                description: The backup volume size.
+                type: string
+              storageClassName:
+                description: the storage class name of pv/pvc binding with the volume.
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: engineimages.longhorn.io
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          name: longhorn-conversion-webhook
+          namespace: longhorn-system
+          path: /v1/webhook/conversion
+          port: 9501
+      conversionReviewVersions:
+      - v1beta2
+      - v1beta1
+  group: longhorn.io
+  names:
+    kind: EngineImage
+    listKind: EngineImageList
+    plural: engineimages
+    shortNames:
+    - lhei
+    singular: engineimage
+  preserveUnknownFields: false
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: State of the engine image
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The Longhorn engine image
+      jsonPath: .spec.image
+      name: Image
+      type: string
+    - description: Number of resources using the engine image
+      jsonPath: .status.refCount
+      name: RefCount
+      type: integer
+    - description: The build date of the engine image
+      jsonPath: .status.buildDate
+      name: BuildDate
+      type: date
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: EngineImage is where Longhorn stores engine image object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: Compatibility of the engine image
+      jsonPath: .status.incompatible
+      name: Incompatible
+      type: boolean
+    - description: State of the engine image
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The Longhorn engine image
+      jsonPath: .spec.image
+      name: Image
+      type: string
+    - description: Number of resources using the engine image
+      jsonPath: .status.refCount
+      name: RefCount
+      type: integer
+    - description: The build date of the engine image
+      jsonPath: .status.buildDate
+      name: BuildDate
+      type: date
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: EngineImage is where Longhorn stores engine image object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: EngineImageSpec defines the desired state of the Longhorn
+              engine image
+            properties:
+              image:
+                minLength: 1
+                type: string
+            required:
+            - image
+            type: object
+          status:
+            description: EngineImageStatus defines the observed state of the Longhorn
+              engine image
+            properties:
+              buildDate:
+                type: string
+              cliAPIMinVersion:
+                type: integer
+              cliAPIVersion:
+                type: integer
+              conditions:
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status
+                        to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about
+                        last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's
+                        last transition.
+                      type: string
+                    status:
+                      description: |-
+                        Status is the status of the condition.
+                        Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                nullable: true
+                type: array
+              controllerAPIMinVersion:
+                type: integer
+              controllerAPIVersion:
+                type: integer
+              dataFormatMinVersion:
+                type: integer
+              dataFormatVersion:
+                type: integer
+              gitCommit:
+                type: string
+              incompatible:
+                type: boolean
+              noRefSince:
+                type: string
+              nodeDeploymentMap:
+                additionalProperties:
+                  type: boolean
+                nullable: true
+                type: object
+              ownerID:
+                type: string
+              refCount:
+                type: integer
+              state:
+                type: string
+              version:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: engines.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: Engine
+    listKind: EngineList
+    plural: engines
+    shortNames:
+    - lhe
+    singular: engine
+  preserveUnknownFields: false
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The current state of the engine
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The node that the engine is on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - description: The instance manager of the engine
+      jsonPath: .status.instanceManagerName
+      name: InstanceManager
+      type: string
+    - description: The current image of the engine
+      jsonPath: .status.currentImage
+      name: Image
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: Engine is where Longhorn stores engine object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The data engine of the engine
+      jsonPath: .spec.dataEngine
+      name: Data Engine
+      type: string
+    - description: The current state of the engine
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The node that the engine is on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - description: The instance manager of the engine
+      jsonPath: .status.instanceManagerName
+      name: InstanceManager
+      type: string
+    - description: The current image of the engine
+      jsonPath: .status.currentImage
+      name: Image
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: Engine is where Longhorn stores engine object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: EngineSpec defines the desired state of the Longhorn engine
+            properties:
+              active:
+                type: boolean
+              backendStoreDriver:
+                description: Deprecated:Replaced by field `dataEngine`.
+                type: string
+              backupVolume:
+                type: string
+              dataEngine:
+                enum:
+                - v1
+                - v2
+                type: string
+              desireState:
+                type: string
+              disableFrontend:
+                type: boolean
+              engineImage:
+                description: 'Deprecated: Replaced by field `image`.'
+                type: string
+              frontend:
+                enum:
+                - blockdev
+                - iscsi
+                - nvmf
+                - ""
+                type: string
+              image:
+                type: string
+              logRequested:
+                type: boolean
+              nodeID:
+                type: string
+              replicaAddressMap:
+                additionalProperties:
+                  type: string
+                type: object
+              requestedBackupRestore:
+                type: string
+              requestedDataSource:
+                type: string
+              revisionCounterDisabled:
+                type: boolean
+              salvageRequested:
+                type: boolean
+              snapshotMaxCount:
+                type: integer
+              snapshotMaxSize:
+                format: int64
+                type: string
+              unmapMarkSnapChainRemovedEnabled:
+                type: boolean
+              upgradedReplicaAddressMap:
+                additionalProperties:
+                  type: string
+                type: object
+              volumeName:
+                type: string
+              volumeSize:
+                format: int64
+                type: string
+            type: object
+          status:
+            description: EngineStatus defines the observed state of the Longhorn engine
+            properties:
+              backupStatus:
+                additionalProperties:
+                  properties:
+                    backupURL:
+                      type: string
+                    error:
+                      type: string
+                    progress:
+                      type: integer
+                    replicaAddress:
+                      type: string
+                    snapshotName:
+                      type: string
+                    state:
+                      type: string
+                  type: object
+                nullable: true
+                type: object
+              cloneStatus:
+                additionalProperties:
+                  properties:
+                    error:
+                      type: string
+                    fromReplicaAddress:
+                      type: string
+                    isCloning:
+                      type: boolean
+                    progress:
+                      type: integer
+                    snapshotName:
+                      type: string
+                    state:
+                      type: string
+                  type: object
+                nullable: true
+                type: object
+              conditions:
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status
+                        to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about
+                        last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's
+                        last transition.
+                      type: string
+                    status:
+                      description: |-
+                        Status is the status of the condition.
+                        Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                nullable: true
+                type: array
+              currentImage:
+                type: string
+              currentReplicaAddressMap:
+                additionalProperties:
+                  type: string
+                nullable: true
+                type: object
+              currentSize:
+                format: int64
+                type: string
+              currentState:
+                type: string
+              endpoint:
+                type: string
+              instanceManagerName:
+                type: string
+              ip:
+                type: string
+              isExpanding:
+                type: boolean
+              lastExpansionError:
+                type: string
+              lastExpansionFailedAt:
+                type: string
+              lastRestoredBackup:
+                type: string
+              logFetched:
+                type: boolean
+              ownerID:
+                type: string
+              port:
+                type: integer
+              purgeStatus:
+                additionalProperties:
+                  properties:
+                    error:
+                      type: string
+                    isPurging:
+                      type: boolean
+                    progress:
+                      type: integer
+                    state:
+                      type: string
+                  type: object
+                nullable: true
+                type: object
+              rebuildStatus:
+                additionalProperties:
+                  properties:
+                    error:
+                      type: string
+                    fromReplicaAddress:
+                      type: string
+                    isRebuilding:
+                      type: boolean
+                    progress:
+                      type: integer
+                    state:
+                      type: string
+                  type: object
+                nullable: true
+                type: object
+              replicaModeMap:
+                additionalProperties:
+                  type: string
+                nullable: true
+                type: object
+              replicaTransitionTimeMap:
+                additionalProperties:
+                  type: string
+                description: |-
+                  ReplicaTransitionTimeMap records the time a replica in ReplicaModeMap transitions from one mode to another (or
+                  from not being in the ReplicaModeMap to being in it). This information is sometimes required by other controllers
+                  (e.g. the volume controller uses it to determine the correct value for replica.Spec.lastHealthyAt).
+                type: object
+              restoreStatus:
+                additionalProperties:
+                  properties:
+                    backupURL:
+                      type: string
+                    currentRestoringBackup:
+                      type: string
+                    error:
+                      type: string
+                    filename:
+                      type: string
+                    isRestoring:
+                      type: boolean
+                    lastRestored:
+                      type: string
+                    progress:
+                      type: integer
+                    state:
+                      type: string
+                  type: object
+                nullable: true
+                type: object
+              salvageExecuted:
+                type: boolean
+              snapshotMaxCount:
+                type: integer
+              snapshotMaxSize:
+                format: int64
+                type: string
+              snapshots:
+                additionalProperties:
+                  properties:
+                    children:
+                      additionalProperties:
+                        type: boolean
+                      nullable: true
+                      type: object
+                    created:
+                      type: string
+                    labels:
+                      additionalProperties:
+                        type: string
+                      nullable: true
+                      type: object
+                    name:
+                      type: string
+                    parent:
+                      type: string
+                    removed:
+                      type: boolean
+                    size:
+                      type: string
+                    usercreated:
+                      type: boolean
+                  type: object
+                nullable: true
+                type: object
+              snapshotsError:
+                type: string
+              started:
+                type: boolean
+              storageIP:
+                type: string
+              unmapMarkSnapChainRemovedEnabled:
+                type: boolean
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: instancemanagers.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: InstanceManager
+    listKind: InstanceManagerList
+    plural: instancemanagers
+    shortNames:
+    - lhim
+    singular: instancemanager
+  preserveUnknownFields: false
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The state of the instance manager
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The type of the instance manager (engine or replica)
+      jsonPath: .spec.type
+      name: Type
+      type: string
+    - description: The node that the instance manager is running on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: InstanceManager is where Longhorn stores instance manager object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The data engine of the instance manager
+      jsonPath: .spec.dataEngine
+      name: Data Engine
+      type: string
+    - description: The state of the instance manager
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The type of the instance manager (engine or replica)
+      jsonPath: .spec.type
+      name: Type
+      type: string
+    - description: The node that the instance manager is running on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: InstanceManager is where Longhorn stores instance manager object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: InstanceManagerSpec defines the desired state of the Longhorn
+              instance manager
+            properties:
+              dataEngine:
+                type: string
+              dataEngineSpec:
+                properties:
+                  v2:
+                    properties:
+                      cpuMask:
+                        type: string
+                    type: object
+                type: object
+              image:
+                type: string
+              nodeID:
+                type: string
+              type:
+                enum:
+                - aio
+                - engine
+                - replica
+                type: string
+            type: object
+          status:
+            description: InstanceManagerStatus defines the observed state of the Longhorn
+              instance manager
+            properties:
+              apiMinVersion:
+                type: integer
+              apiVersion:
+                type: integer
+              backingImages:
+                additionalProperties:
+                  properties:
+                    currentChecksum:
+                      type: string
+                    diskUUID:
+                      type: string
+                    message:
+                      type: string
+                    name:
+                      type: string
+                    progress:
+                      type: integer
+                    size:
+                      format: int64
+                      type: integer
+                    state:
+                      type: string
+                    uuid:
+                      type: string
+                  type: object
+                nullable: true
+                type: object
+              currentState:
+                type: string
+              dataEngineStatus:
+                properties:
+                  v2:
+                    properties:
+                      cpuMask:
+                        type: string
+                    type: object
+                type: object
+              instanceEngines:
+                additionalProperties:
+                  properties:
+                    spec:
+                      properties:
+                        backendStoreDriver:
+                          description: Deprecated:Replaced by field `dataEngine`.
+                          type: string
+                        dataEngine:
+                          type: string
+                        name:
+                          type: string
+                      type: object
+                    status:
+                      properties:
+                        conditions:
+                          additionalProperties:
+                            type: boolean
+                          nullable: true
+                          type: object
+                        endpoint:
+                          type: string
+                        errorMsg:
+                          type: string
+                        listen:
+                          type: string
+                        portEnd:
+                          format: int32
+                          type: integer
+                        portStart:
+                          format: int32
+                          type: integer
+                        resourceVersion:
+                          format: int64
+                          type: integer
+                        state:
+                          type: string
+                        targetPortEnd:
+                          format: int32
+                          type: integer
+                        targetPortStart:
+                          format: int32
+                          type: integer
+                        type:
+                          type: string
+                      type: object
+                  type: object
+                nullable: true
+                type: object
+              instanceReplicas:
+                additionalProperties:
+                  properties:
+                    spec:
+                      properties:
+                        backendStoreDriver:
+                          description: Deprecated:Replaced by field `dataEngine`.
+                          type: string
+                        dataEngine:
+                          type: string
+                        name:
+                          type: string
+                      type: object
+                    status:
+                      properties:
+                        conditions:
+                          additionalProperties:
+                            type: boolean
+                          nullable: true
+                          type: object
+                        endpoint:
+                          type: string
+                        errorMsg:
+                          type: string
+                        listen:
+                          type: string
+                        portEnd:
+                          format: int32
+                          type: integer
+                        portStart:
+                          format: int32
+                          type: integer
+                        resourceVersion:
+                          format: int64
+                          type: integer
+                        state:
+                          type: string
+                        targetPortEnd:
+                          format: int32
+                          type: integer
+                        targetPortStart:
+                          format: int32
+                          type: integer
+                        type:
+                          type: string
+                      type: object
+                  type: object
+                nullable: true
+                type: object
+              instances:
+                additionalProperties:
+                  properties:
+                    spec:
+                      properties:
+                        backendStoreDriver:
+                          description: Deprecated:Replaced by field `dataEngine`.
+                          type: string
+                        dataEngine:
+                          type: string
+                        name:
+                          type: string
+                      type: object
+                    status:
+                      properties:
+                        conditions:
+                          additionalProperties:
+                            type: boolean
+                          nullable: true
+                          type: object
+                        endpoint:
+                          type: string
+                        errorMsg:
+                          type: string
+                        listen:
+                          type: string
+                        portEnd:
+                          format: int32
+                          type: integer
+                        portStart:
+                          format: int32
+                          type: integer
+                        resourceVersion:
+                          format: int64
+                          type: integer
+                        state:
+                          type: string
+                        targetPortEnd:
+                          format: int32
+                          type: integer
+                        targetPortStart:
+                          format: int32
+                          type: integer
+                        type:
+                          type: string
+                      type: object
+                  type: object
+                description: 'Deprecated: Replaced by InstanceEngines and InstanceReplicas'
+                nullable: true
+                type: object
+              ip:
+                type: string
+              ownerID:
+                type: string
+              proxyApiMinVersion:
+                type: integer
+              proxyApiVersion:
+                type: integer
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: nodes.longhorn.io
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          name: longhorn-conversion-webhook
+          namespace: longhorn-system
+          path: /v1/webhook/conversion
+          port: 9501
+      conversionReviewVersions:
+      - v1beta2
+      - v1beta1
+  group: longhorn.io
+  names:
+    kind: Node
+    listKind: NodeList
+    plural: nodes
+    shortNames:
+    - lhn
+    singular: node
+  preserveUnknownFields: false
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: Indicate whether the node is ready
+      jsonPath: .status.conditions['Ready']['status']
+      name: Ready
+      type: string
+    - description: Indicate whether the user disabled/enabled replica scheduling for
+        the node
+      jsonPath: .spec.allowScheduling
+      name: AllowScheduling
+      type: boolean
+    - description: Indicate whether Longhorn can schedule replicas on the node
+      jsonPath: .status.conditions['Schedulable']['status']
+      name: Schedulable
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: Node is where Longhorn stores Longhorn node object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: Indicate whether the node is ready
+      jsonPath: .status.conditions[?(@.type=='Ready')].status
+      name: Ready
+      type: string
+    - description: Indicate whether the user disabled/enabled replica scheduling for
+        the node
+      jsonPath: .spec.allowScheduling
+      name: AllowScheduling
+      type: boolean
+    - description: Indicate whether Longhorn can schedule replicas on the node
+      jsonPath: .status.conditions[?(@.type=='Schedulable')].status
+      name: Schedulable
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: Node is where Longhorn stores Longhorn node object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: NodeSpec defines the desired state of the Longhorn node
+            properties:
+              allowScheduling:
+                type: boolean
+              disks:
+                additionalProperties:
+                  properties:
+                    allowScheduling:
+                      type: boolean
+                    diskDriver:
+                      enum:
+                      - ""
+                      - auto
+                      - aio
+                      type: string
+                    diskType:
+                      enum:
+                      - filesystem
+                      - block
+                      type: string
+                    evictionRequested:
+                      type: boolean
+                    path:
+                      type: string
+                    storageReserved:
+                      format: int64
+                      type: integer
+                    tags:
+                      items:
+                        type: string
+                      type: array
+                  type: object
+                type: object
+              evictionRequested:
+                type: boolean
+              instanceManagerCPURequest:
+                type: integer
+              name:
+                type: string
+              tags:
+                items:
+                  type: string
+                type: array
+            type: object
+          status:
+            description: NodeStatus defines the observed state of the Longhorn node
+            properties:
+              autoEvicting:
+                type: boolean
+              conditions:
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status
+                        to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about
+                        last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's
+                        last transition.
+                      type: string
+                    status:
+                      description: |-
+                        Status is the status of the condition.
+                        Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                nullable: true
+                type: array
+              diskStatus:
+                additionalProperties:
+                  properties:
+                    conditions:
+                      items:
+                        properties:
+                          lastProbeTime:
+                            description: Last time we probed the condition.
+                            type: string
+                          lastTransitionTime:
+                            description: Last time the condition transitioned from
+                              one status to another.
+                            type: string
+                          message:
+                            description: Human-readable message indicating details
+                              about last transition.
+                            type: string
+                          reason:
+                            description: Unique, one-word, CamelCase reason for the
+                              condition's last transition.
+                            type: string
+                          status:
+                            description: |-
+                              Status is the status of the condition.
+                              Can be True, False, Unknown.
+                            type: string
+                          type:
+                            description: Type is the type of the condition.
+                            type: string
+                        type: object
+                      nullable: true
+                      type: array
+                    diskDriver:
+                      type: string
+                    diskName:
+                      type: string
+                    diskPath:
+                      type: string
+                    diskType:
+                      type: string
+                    diskUUID:
+                      type: string
+                    filesystemType:
+                      type: string
+                    instanceManagerName:
+                      type: string
+                    scheduledBackingImage:
+                      additionalProperties:
+                        format: int64
+                        type: integer
+                      nullable: true
+                      type: object
+                    scheduledReplica:
+                      additionalProperties:
+                        format: int64
+                        type: integer
+                      nullable: true
+                      type: object
+                    storageAvailable:
+                      format: int64
+                      type: integer
+                    storageMaximum:
+                      format: int64
+                      type: integer
+                    storageScheduled:
+                      format: int64
+                      type: integer
+                  type: object
+                nullable: true
+                type: object
+              region:
+                type: string
+              snapshotCheckStatus:
+                properties:
+                  lastPeriodicCheckedAt:
+                    format: date-time
+                    type: string
+                type: object
+              zone:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: orphans.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: Orphan
+    listKind: OrphanList
+    plural: orphans
+    shortNames:
+    - lho
+    singular: orphan
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The type of the orphan
+      jsonPath: .spec.orphanType
+      name: Type
+      type: string
+    - description: The node that the orphan is on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: Orphan is where Longhorn stores orphan object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: OrphanSpec defines the desired state of the Longhorn orphaned
+              data
+            properties:
+              nodeID:
+                description: The node ID on which the controller is responsible to
+                  reconcile this orphan CR.
+                type: string
+              orphanType:
+                description: |-
+                  The type of the orphaned data.
+                  Can be "replica".
+                type: string
+              parameters:
+                additionalProperties:
+                  type: string
+                description: The parameters of the orphaned data
+                type: object
+            type: object
+          status:
+            description: OrphanStatus defines the observed state of the Longhorn orphaned
+              data
+            properties:
+              conditions:
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status
+                        to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about
+                        last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's
+                        last transition.
+                      type: string
+                    status:
+                      description: |-
+                        Status is the status of the condition.
+                        Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                nullable: true
+                type: array
+              ownerID:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: recurringjobs.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: RecurringJob
+    listKind: RecurringJobList
+    plural: recurringjobs
+    shortNames:
+    - lhrj
+    singular: recurringjob
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: Sets groupings to the jobs. When set to "default" group will be
+        added to the volume label when no other job label exist in volume
+      jsonPath: .spec.groups
+      name: Groups
+      type: string
+    - description: Should be one of "backup" or "snapshot"
+      jsonPath: .spec.task
+      name: Task
+      type: string
+    - description: The cron expression represents recurring job scheduling
+      jsonPath: .spec.cron
+      name: Cron
+      type: string
+    - description: The number of snapshots/backups to keep for the volume
+      jsonPath: .spec.retain
+      name: Retain
+      type: integer
+    - description: The concurrent job to run by each cron job
+      jsonPath: .spec.concurrency
+      name: Concurrency
+      type: integer
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    - description: Specify the labels
+      jsonPath: .spec.labels
+      name: Labels
+      type: string
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: RecurringJob is where Longhorn stores recurring job object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: Sets groupings to the jobs. When set to "default" group will be
+        added to the volume label when no other job label exist in volume
+      jsonPath: .spec.groups
+      name: Groups
+      type: string
+    - description: Should be one of "snapshot", "snapshot-force-create", "snapshot-cleanup",
+        "snapshot-delete", "backup", "backup-force-create" or "filesystem-trim"
+      jsonPath: .spec.task
+      name: Task
+      type: string
+    - description: The cron expression represents recurring job scheduling
+      jsonPath: .spec.cron
+      name: Cron
+      type: string
+    - description: The number of snapshots/backups to keep for the volume
+      jsonPath: .spec.retain
+      name: Retain
+      type: integer
+    - description: The concurrent job to run by each cron job
+      jsonPath: .spec.concurrency
+      name: Concurrency
+      type: integer
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    - description: Specify the labels
+      jsonPath: .spec.labels
+      name: Labels
+      type: string
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: RecurringJob is where Longhorn stores recurring job object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: RecurringJobSpec defines the desired state of the Longhorn
+              recurring job
+            properties:
+              concurrency:
+                description: The concurrency of taking the snapshot/backup.
+                type: integer
+              cron:
+                description: The cron setting.
+                type: string
+              groups:
+                description: The recurring job group.
+                items:
+                  type: string
+                type: array
+              labels:
+                additionalProperties:
+                  type: string
+                description: The label of the snapshot/backup.
+                type: object
+              name:
+                description: The recurring job name.
+                type: string
+              parameters:
+                additionalProperties:
+                  type: string
+                description: |-
+                  The parameters of the snapshot/backup.
+                  Support parameters: "full-backup-interval".
+                type: object
+              retain:
+                description: The retain count of the snapshot/backup.
+                type: integer
+              task:
+                description: |-
+                  The recurring job task.
+                  Can be "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create" or "filesystem-trim"
+                enum:
+                - snapshot
+                - snapshot-force-create
+                - snapshot-cleanup
+                - snapshot-delete
+                - backup
+                - backup-force-create
+                - filesystem-trim
+                type: string
+            type: object
+          status:
+            description: RecurringJobStatus defines the observed state of the Longhorn
+              recurring job
+            properties:
+              executionCount:
+                description: The number of jobs that have been triggered.
+                type: integer
+              ownerID:
+                description: The owner ID which is responsible to reconcile this recurring
+                  job CR.
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: replicas.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: Replica
+    listKind: ReplicaList
+    plural: replicas
+    shortNames:
+    - lhr
+    singular: replica
+  preserveUnknownFields: false
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The current state of the replica
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The node that the replica is on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - description: The disk that the replica is on
+      jsonPath: .spec.diskID
+      name: Disk
+      type: string
+    - description: The instance manager of the replica
+      jsonPath: .status.instanceManagerName
+      name: InstanceManager
+      type: string
+    - description: The current image of the replica
+      jsonPath: .status.currentImage
+      name: Image
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: Replica is where Longhorn stores replica object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The data engine of the replica
+      jsonPath: .spec.dataEngine
+      name: Data Engine
+      type: string
+    - description: The current state of the replica
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The node that the replica is on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - description: The disk that the replica is on
+      jsonPath: .spec.diskID
+      name: Disk
+      type: string
+    - description: The instance manager of the replica
+      jsonPath: .status.instanceManagerName
+      name: InstanceManager
+      type: string
+    - description: The current image of the replica
+      jsonPath: .status.currentImage
+      name: Image
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: Replica is where Longhorn stores replica object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: ReplicaSpec defines the desired state of the Longhorn replica
+            properties:
+              active:
+                type: boolean
+              backendStoreDriver:
+                description: Deprecated:Replaced by field `dataEngine`.
+                type: string
+              backingImage:
+                type: string
+              dataDirectoryName:
+                type: string
+              dataEngine:
+                enum:
+                - v1
+                - v2
+                type: string
+              desireState:
+                type: string
+              diskID:
+                type: string
+              diskPath:
+                type: string
+              engineImage:
+                description: 'Deprecated: Replaced by field `image`.'
+                type: string
+              engineName:
+                type: string
+              evictionRequested:
+                type: boolean
+              failedAt:
+                description: |-
+                  FailedAt is set when a running replica fails or when a running engine is unable to use a replica for any reason.
+                  FailedAt indicates the time the failure occurred. When FailedAt is set, a replica is likely to have useful
+                  (though possibly stale) data. A replica with FailedAt set must be rebuilt from a non-failed replica (or it can
+                  be used in a salvage if all replicas are failed). FailedAt is cleared before a rebuild or salvage. FailedAt may
+                  be later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume
+                  controller acknowledges the change.
+                type: string
+              hardNodeAffinity:
+                type: string
+              healthyAt:
+                description: |-
+                  HealthyAt is set the first time a replica becomes read/write in an engine after creation or rebuild. HealthyAt
+                  indicates the time the last successful rebuild occurred. When HealthyAt is set, a replica is likely to have
+                  useful (though possibly stale) data. HealthyAt is cleared before a rebuild. HealthyAt may be later than the
+                  corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller
+                  acknowledges the change.
+                type: string
+              image:
+                type: string
+              lastFailedAt:
+                description: |-
+                  LastFailedAt is always set at the same time as FailedAt. Unlike FailedAt, LastFailedAt is never cleared.
+                  LastFailedAt is not a reliable indicator of the state of a replica's data. For example, a replica with
+                  LastFailedAt may already be healthy and in use again. However, because it is never cleared, it can be compared to
+                  LastHealthyAt to help prevent dangerous replica deletion in some corner cases. LastFailedAt may be later than the
+                  corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller
+                  acknowledges the change.
+                type: string
+              lastHealthyAt:
+                description: |-
+                  LastHealthyAt is set every time a replica becomes read/write in an engine. Unlike HealthyAt, LastHealthyAt is
+                  never cleared. LastHealthyAt is not a reliable indicator of the state of a replica's data. For example, a
+                  replica with LastHealthyAt set may be in the middle of a rebuild. However, because it is never cleared, it can be
+                  compared to LastFailedAt to help prevent dangerous replica deletion in some corner cases. LastHealthyAt may be
+                  later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume
+                  controller acknowledges the change.
+                type: string
+              logRequested:
+                type: boolean
+              migrationEngineName:
+                description: |-
+                  MigrationEngineName is indicating the migrating engine which current connected to this replica. This is only
+                  used for live migration of v2 data engine
+                type: string
+              nodeID:
+                type: string
+              rebuildRetryCount:
+                type: integer
+              revisionCounterDisabled:
+                type: boolean
+              salvageRequested:
+                type: boolean
+              snapshotMaxCount:
+                type: integer
+              snapshotMaxSize:
+                format: int64
+                type: string
+              unmapMarkDiskChainRemovedEnabled:
+                type: boolean
+              volumeName:
+                type: string
+              volumeSize:
+                format: int64
+                type: string
+            type: object
+          status:
+            description: ReplicaStatus defines the observed state of the Longhorn
+              replica
+            properties:
+              conditions:
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status
+                        to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about
+                        last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's
+                        last transition.
+                      type: string
+                    status:
+                      description: |-
+                        Status is the status of the condition.
+                        Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                nullable: true
+                type: array
+              currentImage:
+                type: string
+              currentState:
+                type: string
+              evictionRequested:
+                description: 'Deprecated: Replaced by field `spec.evictionRequested`.'
+                type: boolean
+              instanceManagerName:
+                type: string
+              ip:
+                type: string
+              logFetched:
+                type: boolean
+              ownerID:
+                type: string
+              port:
+                type: integer
+              salvageExecuted:
+                type: boolean
+              started:
+                type: boolean
+              storageIP:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: settings.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: Setting
+    listKind: SettingList
+    plural: settings
+    shortNames:
+    - lhs
+    singular: setting
+  preserveUnknownFields: false
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The value of the setting
+      jsonPath: .value
+      name: Value
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: Setting is where Longhorn stores setting object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          value:
+            type: string
+        required:
+        - value
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The value of the setting
+      jsonPath: .value
+      name: Value
+      type: string
+    - description: The setting is applied
+      jsonPath: .status.applied
+      name: Applied
+      type: boolean
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: Setting is where Longhorn stores setting object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          status:
+            description: The status of the setting.
+            properties:
+              applied:
+                description: The setting is applied.
+                type: boolean
+            required:
+            - applied
+            type: object
+          value:
+            description: The value of the setting.
+            type: string
+        required:
+        - value
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: sharemanagers.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: ShareManager
+    listKind: ShareManagerList
+    plural: sharemanagers
+    shortNames:
+    - lhsm
+    singular: sharemanager
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The state of the share manager
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The node that the share manager is owned by
+      jsonPath: .status.ownerID
+      name: Node
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: ShareManager is where Longhorn stores share manager object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The state of the share manager
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The node that the share manager is owned by
+      jsonPath: .status.ownerID
+      name: Node
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: ShareManager is where Longhorn stores share manager object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: ShareManagerSpec defines the desired state of the Longhorn
+              share manager
+            properties:
+              image:
+                description: Share manager image used for creating a share manager
+                  pod
+                type: string
+            type: object
+          status:
+            description: ShareManagerStatus defines the observed state of the Longhorn
+              share manager
+            properties:
+              endpoint:
+                description: NFS endpoint that can access the mounted filesystem of
+                  the volume
+                type: string
+              ownerID:
+                description: The node ID on which the controller is responsible to
+                  reconcile this share manager resource
+                type: string
+              state:
+                description: The state of the share manager resource
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: snapshots.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: Snapshot
+    listKind: SnapshotList
+    plural: snapshots
+    shortNames:
+    - lhsnap
+    singular: snapshot
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The volume that this snapshot belongs to
+      jsonPath: .spec.volume
+      name: Volume
+      type: string
+    - description: Timestamp when the point-in-time snapshot was taken
+      jsonPath: .status.creationTime
+      name: CreationTime
+      type: string
+    - description: Indicates if the snapshot is ready to be used to restore/backup
+        a volume
+      jsonPath: .status.readyToUse
+      name: ReadyToUse
+      type: boolean
+    - description: Represents the minimum size of volume required to rehydrate from
+        this snapshot
+      jsonPath: .status.restoreSize
+      name: RestoreSize
+      type: string
+    - description: The actual size of the snapshot
+      jsonPath: .status.size
+      name: Size
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: Snapshot is the Schema for the snapshots API
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: SnapshotSpec defines the desired state of Longhorn Snapshot
+            properties:
+              createSnapshot:
+                description: require creating a new snapshot
+                type: boolean
+              labels:
+                additionalProperties:
+                  type: string
+                description: The labels of snapshot
+                nullable: true
+                type: object
+              volume:
+                description: |-
+                  the volume that this snapshot belongs to.
+                  This field is immutable after creation.
+                type: string
+            required:
+            - volume
+            type: object
+          status:
+            description: SnapshotStatus defines the observed state of Longhorn Snapshot
+            properties:
+              checksum:
+                type: string
+              children:
+                additionalProperties:
+                  type: boolean
+                nullable: true
+                type: object
+              creationTime:
+                type: string
+              error:
+                type: string
+              labels:
+                additionalProperties:
+                  type: string
+                nullable: true
+                type: object
+              markRemoved:
+                type: boolean
+              ownerID:
+                type: string
+              parent:
+                type: string
+              readyToUse:
+                type: boolean
+              restoreSize:
+                format: int64
+                type: integer
+              size:
+                format: int64
+                type: integer
+              userCreated:
+                type: boolean
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: supportbundles.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: SupportBundle
+    listKind: SupportBundleList
+    plural: supportbundles
+    shortNames:
+    - lhbundle
+    singular: supportbundle
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The state of the support bundle
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The issue URL
+      jsonPath: .spec.issueURL
+      name: Issue
+      type: string
+    - description: A brief description of the issue
+      jsonPath: .spec.description
+      name: Description
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: SupportBundle is where Longhorn stores support bundle object
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: SupportBundleSpec defines the desired state of the Longhorn
+              SupportBundle
+            properties:
+              description:
+                description: A brief description of the issue
+                type: string
+              issueURL:
+                description: The issue URL
+                nullable: true
+                type: string
+              nodeID:
+                description: The preferred responsible controller node ID.
+                type: string
+            required:
+            - description
+            type: object
+          status:
+            description: SupportBundleStatus defines the observed state of the Longhorn
+              SupportBundle
+            properties:
+              conditions:
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status
+                        to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about
+                        last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's
+                        last transition.
+                      type: string
+                    status:
+                      description: |-
+                        Status is the status of the condition.
+                        Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                type: array
+              filename:
+                type: string
+              filesize:
+                format: int64
+                type: integer
+              image:
+                description: The support bundle manager image
+                type: string
+              managerIP:
+                description: The support bundle manager IP
+                type: string
+              ownerID:
+                description: The current responsible controller node ID
+                type: string
+              progress:
+                type: integer
+              state:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: systembackups.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: SystemBackup
+    listKind: SystemBackupList
+    plural: systembackups
+    shortNames:
+    - lhsb
+    singular: systembackup
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The system backup Longhorn version
+      jsonPath: .status.version
+      name: Version
+      type: string
+    - description: The system backup state
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The system backup creation time
+      jsonPath: .status.createdAt
+      name: Created
+      type: string
+    - description: The last time that the system backup was synced into the cluster
+      jsonPath: .status.lastSyncedAt
+      name: LastSyncedAt
+      type: string
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: SystemBackup is where Longhorn stores system backup object
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: SystemBackupSpec defines the desired state of the Longhorn
+              SystemBackup
+            properties:
+              volumeBackupPolicy:
+                description: |-
+                  The create volume backup policy
+                  Can be "if-not-present", "always" or "disabled"
+                nullable: true
+                type: string
+            type: object
+          status:
+            description: SystemBackupStatus defines the observed state of the Longhorn
+              SystemBackup
+            properties:
+              conditions:
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status
+                        to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about
+                        last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's
+                        last transition.
+                      type: string
+                    status:
+                      description: |-
+                        Status is the status of the condition.
+                        Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                nullable: true
+                type: array
+              createdAt:
+                description: The system backup creation time.
+                format: date-time
+                type: string
+              gitCommit:
+                description: The saved Longhorn manager git commit.
+                nullable: true
+                type: string
+              lastSyncedAt:
+                description: The last time that the system backup was synced into
+                  the cluster.
+                format: date-time
+                nullable: true
+                type: string
+              managerImage:
+                description: The saved manager image.
+                type: string
+              ownerID:
+                description: The node ID of the responsible controller to reconcile
+                  this SystemBackup.
+                type: string
+              state:
+                description: The system backup state.
+                type: string
+              version:
+                description: The saved Longhorn version.
+                nullable: true
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: systemrestores.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: SystemRestore
+    listKind: SystemRestoreList
+    plural: systemrestores
+    shortNames:
+    - lhsr
+    singular: systemrestore
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The system restore state
+      jsonPath: .status.state
+      name: State
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: SystemRestore is where Longhorn stores system restore object
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: SystemRestoreSpec defines the desired state of the Longhorn
+              SystemRestore
+            properties:
+              systemBackup:
+                description: The system backup name in the object store.
+                type: string
+            required:
+            - systemBackup
+            type: object
+          status:
+            description: SystemRestoreStatus defines the observed state of the Longhorn
+              SystemRestore
+            properties:
+              conditions:
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status
+                        to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about
+                        last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's
+                        last transition.
+                      type: string
+                    status:
+                      description: |-
+                        Status is the status of the condition.
+                        Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                nullable: true
+                type: array
+              ownerID:
+                description: The node ID of the responsible controller to reconcile
+                  this SystemRestore.
+                type: string
+              sourceURL:
+                description: The source system backup URL.
+                type: string
+              state:
+                description: The system restore state.
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: volumeattachments.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: VolumeAttachment
+    listKind: VolumeAttachmentList
+    plural: volumeattachments
+    shortNames:
+    - lhva
+    singular: volumeattachment
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: VolumeAttachment stores attachment information of a Longhorn
+          volume
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: VolumeAttachmentSpec defines the desired state of Longhorn
+              VolumeAttachment
+            properties:
+              attachmentTickets:
+                additionalProperties:
+                  properties:
+                    generation:
+                      description: |-
+                        A sequence number representing a specific generation of the desired state.
+                        Populated by the system. Read-only.
+                      format: int64
+                      type: integer
+                    id:
+                      description: The unique ID of this attachment. Used to differentiate
+                        different attachments of the same volume.
+                      type: string
+                    nodeID:
+                      description: The node that this attachment is requesting
+                      type: string
+                    parameters:
+                      additionalProperties:
+                        type: string
+                      description: Optional additional parameter for this attachment
+                      type: object
+                    type:
+                      type: string
+                  type: object
+                type: object
+              volume:
+                description: The name of Longhorn volume of this VolumeAttachment
+                type: string
+            required:
+            - volume
+            type: object
+          status:
+            description: VolumeAttachmentStatus defines the observed state of Longhorn
+              VolumeAttachment
+            properties:
+              attachmentTicketStatuses:
+                additionalProperties:
+                  properties:
+                    conditions:
+                      description: Record any error when trying to fulfill this attachment
+                      items:
+                        properties:
+                          lastProbeTime:
+                            description: Last time we probed the condition.
+                            type: string
+                          lastTransitionTime:
+                            description: Last time the condition transitioned from
+                              one status to another.
+                            type: string
+                          message:
+                            description: Human-readable message indicating details
+                              about last transition.
+                            type: string
+                          reason:
+                            description: Unique, one-word, CamelCase reason for the
+                              condition's last transition.
+                            type: string
+                          status:
+                            description: |-
+                              Status is the status of the condition.
+                              Can be True, False, Unknown.
+                            type: string
+                          type:
+                            description: Type is the type of the condition.
+                            type: string
+                        type: object
+                      nullable: true
+                      type: array
+                    generation:
+                      description: |-
+                        A sequence number representing a specific generation of the desired state.
+                        Populated by the system. Read-only.
+                      format: int64
+                      type: integer
+                    id:
+                      description: The unique ID of this attachment. Used to differentiate
+                        different attachments of the same volume.
+                      type: string
+                    satisfied:
+                      description: Indicate whether this attachment ticket has been
+                        satisfied
+                      type: boolean
+                  required:
+                  - conditions
+                  - satisfied
+                  type: object
+                type: object
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    longhorn-manager: ""
+  name: volumes.longhorn.io
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          name: longhorn-conversion-webhook
+          namespace: longhorn-system
+          path: /v1/webhook/conversion
+          port: 9501
+      conversionReviewVersions:
+      - v1beta2
+      - v1beta1
+  group: longhorn.io
+  names:
+    kind: Volume
+    listKind: VolumeList
+    plural: volumes
+    shortNames:
+    - lhv
+    singular: volume
+  preserveUnknownFields: false
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The state of the volume
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The robustness of the volume
+      jsonPath: .status.robustness
+      name: Robustness
+      type: string
+    - description: The scheduled condition of the volume
+      jsonPath: .status.conditions['scheduled']['status']
+      name: Scheduled
+      type: string
+    - description: The size of the volume
+      jsonPath: .spec.size
+      name: Size
+      type: string
+    - description: The node that the volume is currently attaching to
+      jsonPath: .status.currentNodeID
+      name: Node
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: Volume is where Longhorn stores volume object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The data engine of the volume
+      jsonPath: .spec.dataEngine
+      name: Data Engine
+      type: string
+    - description: The state of the volume
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The robustness of the volume
+      jsonPath: .status.robustness
+      name: Robustness
+      type: string
+    - description: The scheduled condition of the volume
+      jsonPath: .status.conditions[?(@.type=='Schedulable')].status
+      name: Scheduled
+      type: string
+    - description: The size of the volume
+      jsonPath: .spec.size
+      name: Size
+      type: string
+    - description: The node that the volume is currently attaching to
+      jsonPath: .status.currentNodeID
+      name: Node
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: Volume is where Longhorn stores volume object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: VolumeSpec defines the desired state of the Longhorn volume
+            properties:
+              Standby:
+                type: boolean
+              accessMode:
+                enum:
+                - rwo
+                - rwx
+                type: string
+              backendStoreDriver:
+                description: Deprecated:Replaced by field `dataEngine`.'
+                type: string
+              backingImage:
+                type: string
+              backupCompressionMethod:
+                enum:
+                - none
+                - lz4
+                - gzip
+                type: string
+              backupTargetName:
+                description: The backup target name that the volume will be backed
+                  up to or is synced.
+                type: string
+              dataEngine:
+                enum:
+                - v1
+                - v2
+                type: string
+              dataLocality:
+                enum:
+                - disabled
+                - best-effort
+                - strict-local
+                type: string
+              dataSource:
+                type: string
+              disableFrontend:
+                type: boolean
+              diskSelector:
+                items:
+                  type: string
+                type: array
+              encrypted:
+                type: boolean
+              engineImage:
+                description: 'Deprecated: Replaced by field `image`.'
+                type: string
+              freezeFilesystemForSnapshot:
+                description: Setting that freezes the filesystem on the root partition
+                  before a snapshot is created.
+                enum:
+                - ignored
+                - enabled
+                - disabled
+                type: string
+              fromBackup:
+                type: string
+              frontend:
+                enum:
+                - blockdev
+                - iscsi
+                - nvmf
+                - ""
+                type: string
+              image:
+                type: string
+              lastAttachedBy:
+                type: string
+              migratable:
+                type: boolean
+              migrationNodeID:
+                type: string
+              nodeID:
+                type: string
+              nodeSelector:
+                items:
+                  type: string
+                type: array
+              numberOfReplicas:
+                type: integer
+              replicaAutoBalance:
+                enum:
+                - ignored
+                - disabled
+                - least-effort
+                - best-effort
+                type: string
+              replicaDiskSoftAntiAffinity:
+                description: Replica disk soft anti affinity of the volume. Set enabled
+                  to allow replicas to be scheduled in the same disk.
+                enum:
+                - ignored
+                - enabled
+                - disabled
+                type: string
+              replicaSoftAntiAffinity:
+                description: Replica soft anti affinity of the volume. Set enabled
+                  to allow replicas to be scheduled on the same node.
+                enum:
+                - ignored
+                - enabled
+                - disabled
+                type: string
+              replicaZoneSoftAntiAffinity:
+                description: Replica zone soft anti affinity of the volume. Set enabled
+                  to allow replicas to be scheduled in the same zone.
+                enum:
+                - ignored
+                - enabled
+                - disabled
+                type: string
+              restoreVolumeRecurringJob:
+                enum:
+                - ignored
+                - enabled
+                - disabled
+                type: string
+              revisionCounterDisabled:
+                type: boolean
+              size:
+                format: int64
+                type: string
+              snapshotDataIntegrity:
+                enum:
+                - ignored
+                - disabled
+                - enabled
+                - fast-check
+                type: string
+              snapshotMaxCount:
+                type: integer
+              snapshotMaxSize:
+                format: int64
+                type: string
+              staleReplicaTimeout:
+                type: integer
+              unmapMarkSnapChainRemoved:
+                enum:
+                - ignored
+                - disabled
+                - enabled
+                type: string
+            type: object
+          status:
+            description: VolumeStatus defines the observed state of the Longhorn volume
+            properties:
+              actualSize:
+                format: int64
+                type: integer
+              cloneStatus:
+                properties:
+                  attemptCount:
+                    type: integer
+                  nextAllowedAttemptAt:
+                    type: string
+                  snapshot:
+                    type: string
+                  sourceVolume:
+                    type: string
+                  state:
+                    type: string
+                type: object
+              conditions:
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status
+                        to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about
+                        last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's
+                        last transition.
+                      type: string
+                    status:
+                      description: |-
+                        Status is the status of the condition.
+                        Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                nullable: true
+                type: array
+              currentImage:
+                type: string
+              currentMigrationNodeID:
+                description: the node that this volume is currently migrating to
+                type: string
+              currentNodeID:
+                type: string
+              expansionRequired:
+                type: boolean
+              frontendDisabled:
+                type: boolean
+              isStandby:
+                type: boolean
+              kubernetesStatus:
+                properties:
+                  lastPVCRefAt:
+                    type: string
+                  lastPodRefAt:
+                    type: string
+                  namespace:
+                    description: determine if PVC/Namespace is history or not
+                    type: string
+                  pvName:
+                    type: string
+                  pvStatus:
+                    type: string
+                  pvcName:
+                    type: string
+                  workloadsStatus:
+                    description: determine if Pod/Workload is history or not
+                    items:
+                      properties:
+                        podName:
+                          type: string
+                        podStatus:
+                          type: string
+                        workloadName:
+                          type: string
+                        workloadType:
+                          type: string
+                      type: object
+                    nullable: true
+                    type: array
+                type: object
+              lastBackup:
+                type: string
+              lastBackupAt:
+                type: string
+              lastDegradedAt:
+                type: string
+              ownerID:
+                type: string
+              pendingNodeID:
+                description: Deprecated.
+                type: string
+              remountRequestedAt:
+                type: string
+              restoreInitiated:
+                type: boolean
+              restoreRequired:
+                type: boolean
+              robustness:
+                type: string
+              shareEndpoint:
+                type: string
+              shareState:
+                type: string
+              state:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+# Source: longhorn/templates/clusterrole.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: longhorn-role
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+rules:
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - "*"
+- apiGroups: [""]
+  resources: ["pods", "events", "persistentvolumes", "persistentvolumeclaims","persistentvolumeclaims/status", "nodes", "proxy/nodes", "pods/log", "secrets", "services", "endpoints", "configmaps", "serviceaccounts"]
+  verbs: ["*"]
+- apiGroups: [""]
+  resources: ["namespaces"]
+  verbs: ["get", "list"]
+- apiGroups: ["apps"]
+  resources: ["daemonsets", "statefulsets", "deployments"]
+  verbs: ["*"]
+- apiGroups: ["batch"]
+  resources: ["jobs", "cronjobs"]
+  verbs: ["*"]
+- apiGroups: ["policy"]
+  resources: ["poddisruptionbudgets", "podsecuritypolicies"]
+  verbs: ["*"]
+- apiGroups: ["scheduling.k8s.io"]
+  resources: ["priorityclasses"]
+  verbs: ["watch", "list"]
+- apiGroups: ["storage.k8s.io"]
+  resources: ["storageclasses", "volumeattachments", "volumeattachments/status", "csinodes", "csidrivers"]
+  verbs: ["*"]
+- apiGroups: ["snapshot.storage.k8s.io"]
+  resources: ["volumesnapshotclasses", "volumesnapshots", "volumesnapshotcontents", "volumesnapshotcontents/status"]
+  verbs: ["*"]
+- apiGroups: ["longhorn.io"]
+  resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings", "settings/status",
+              "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status",
+              "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status",
+              "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status",
+              "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status",
+              "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status",
+              "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status",
+              "volumeattachments", "volumeattachments/status", "backupbackingimages", "backupbackingimages/status"]
+  verbs: ["*"]
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["*"]
+- apiGroups: ["metrics.k8s.io"]
+  resources: ["pods", "nodes"]
+  verbs: ["get", "list"]
+- apiGroups: ["apiregistration.k8s.io"]
+  resources: ["apiservices"]
+  verbs: ["list", "watch"]
+- apiGroups: ["admissionregistration.k8s.io"]
+  resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"]
+  verbs: ["get", "list", "create", "patch", "delete"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["roles", "rolebindings", "clusterrolebindings", "clusterroles"]
+  verbs: ["*"]
+---
+# Source: longhorn/templates/clusterrolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: longhorn-bind
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: longhorn-role
+subjects:
+- kind: ServiceAccount
+  name: longhorn-service-account
+  namespace: longhorn-system
+---
+# Source: longhorn/templates/clusterrolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: longhorn-support-bundle
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: longhorn-support-bundle
+  namespace: longhorn-system
+---
+# Source: longhorn/templates/daemonset-sa.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    app: longhorn-manager
+  name: longhorn-backend
+  namespace: longhorn-system
+spec:
+  type: ClusterIP
+  selector:
+    app: longhorn-manager
+  ports:
+  - name: manager
+    port: 9500
+    targetPort: manager
+---
+# Source: longhorn/templates/deployment-ui.yaml
+kind: Service
+apiVersion: v1
+metadata:
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    app: longhorn-ui
+  name: longhorn-frontend
+  namespace: longhorn-system
+spec:
+  type: ClusterIP
+  selector:
+    app: longhorn-ui
+  ports:
+  - name: http
+    port: 80
+    targetPort: http
+    nodePort: null
+---
+# Source: longhorn/templates/services.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    app: longhorn-conversion-webhook
+  name: longhorn-conversion-webhook
+  namespace: longhorn-system
+spec:
+  type: ClusterIP
+  selector:
+    longhorn.io/conversion-webhook: longhorn-conversion-webhook
+  ports:
+  - name: conversion-webhook
+    port: 9501
+    targetPort: conversion-wh
+---
+# Source: longhorn/templates/services.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    app: longhorn-admission-webhook
+  name: longhorn-admission-webhook
+  namespace: longhorn-system
+spec:
+  type: ClusterIP
+  selector:
+    longhorn.io/admission-webhook: longhorn-admission-webhook
+  ports:
+  - name: admission-webhook
+    port: 9502
+    targetPort: admission-wh
+---
+# Source: longhorn/templates/services.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    app: longhorn-recovery-backend
+  name: longhorn-recovery-backend
+  namespace: longhorn-system
+spec:
+  type: ClusterIP
+  selector:
+    longhorn.io/recovery-backend: longhorn-recovery-backend
+  ports:
+  - name: recovery-backend
+    port: 9503
+    targetPort: recov-backend
+---
+# Source: longhorn/templates/daemonset-sa.yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    app: longhorn-manager
+  name: longhorn-manager
+  namespace: longhorn-system
+spec:
+  selector:
+    matchLabels:
+      app: longhorn-manager
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: longhorn
+        helm.sh/chart: longhorn-1.8.0
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/instance: longhorn
+        app.kubernetes.io/version: v1.8.0
+        app: longhorn-manager
+    spec:
+      containers:
+      - name: longhorn-manager
+        image: longhornio/longhorn-manager:v1.8.0
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          privileged: true
+        command:
+        - longhorn-manager
+        - -d
+        - daemon
+        - --engine-image
+        - "longhornio/longhorn-engine:v1.8.0"
+        - --instance-manager-image
+        - "longhornio/longhorn-instance-manager:v1.8.0"
+        - --share-manager-image
+        - "longhornio/longhorn-share-manager:v1.8.0"
+        - --backing-image-manager-image
+        - "longhornio/backing-image-manager:v1.8.0"
+        - --support-bundle-manager-image
+        - "longhornio/support-bundle-kit:v0.0.49"
+        - --manager-image
+        - "longhornio/longhorn-manager:v1.8.0"
+        - --service-account
+        - longhorn-service-account
+        - --upgrade-version-check
+        ports:
+        - containerPort: 9500
+          name: manager
+        - containerPort: 9501
+          name: conversion-wh
+        - containerPort: 9502
+          name: admission-wh
+        - containerPort: 9503
+          name: recov-backend
+        readinessProbe:
+          httpGet:
+            path: /v1/healthz
+            port: 9501
+            scheme: HTTPS
+        volumeMounts:
+        - name: boot
+          mountPath: /host/boot/
+          readOnly: true
+        - name: dev
+          mountPath: /host/dev/
+        - name: proc
+          mountPath: /host/proc/
+          readOnly: true
+        - name: etc
+          mountPath: /host/etc/
+          readOnly: true
+        - name: longhorn
+          mountPath: /var/lib/longhorn/
+          mountPropagation: Bidirectional
+        - name: longhorn-grpc-tls
+          mountPath: /tls-files/
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+      - name: pre-pull-share-manager-image
+        imagePullPolicy: IfNotPresent
+        image: longhornio/longhorn-share-manager:v1.8.0
+        command: ["sh", "-c", "echo share-manager image pulled && sleep infinity"]
+      volumes:
+      - name: boot
+        hostPath:
+          path: /boot/
+      - name: dev
+        hostPath:
+          path: /dev/
+      - name: proc
+        hostPath:
+          path: /proc/
+      - name: etc
+        hostPath:
+          path: /etc/
+      - name: longhorn
+        hostPath:
+          path: /var/lib/longhorn/
+      - name: longhorn-grpc-tls
+        secret:
+          secretName: longhorn-grpc-tls
+          optional: true
+      priorityClassName: "longhorn-critical"
+      serviceAccountName: longhorn-service-account
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: "100%"
+---
+# Source: longhorn/templates/deployment-driver.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: longhorn-driver-deployer
+  namespace: longhorn-system
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: longhorn-driver-deployer
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: longhorn
+        helm.sh/chart: longhorn-1.8.0
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/instance: longhorn
+        app.kubernetes.io/version: v1.8.0
+        app: longhorn-driver-deployer
+    spec:
+      initContainers:
+        - name: wait-longhorn-manager
+          image: longhornio/longhorn-manager:v1.8.0
+          command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done']
+      containers:
+        - name: longhorn-driver-deployer
+          image: longhornio/longhorn-manager:v1.8.0
+          imagePullPolicy: IfNotPresent
+          command:
+          - longhorn-manager
+          - -d
+          - deploy-driver
+          - --manager-image
+          - "longhornio/longhorn-manager:v1.8.0"
+          - --manager-url
+          - http://longhorn-backend:9500/v1
+          env:
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+          - name: SERVICE_ACCOUNT
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.serviceAccountName
+          - name: CSI_ATTACHER_IMAGE
+            value: "longhornio/csi-attacher:v4.8.0"
+          - name: CSI_PROVISIONER_IMAGE
+            value: "longhornio/csi-provisioner:v5.1.0-20241220"
+          - name: CSI_NODE_DRIVER_REGISTRAR_IMAGE
+            value: "longhornio/csi-node-driver-registrar:v2.13.0"
+          - name: CSI_RESIZER_IMAGE
+            value: "longhornio/csi-resizer:v1.13.1"
+          - name: CSI_SNAPSHOTTER_IMAGE
+            value: "longhornio/csi-snapshotter:v8.2.0"
+          - name: CSI_LIVENESS_PROBE_IMAGE
+            value: "longhornio/livenessprobe:v2.15.0"
+          - name: CSI_ATTACHER_REPLICA_COUNT
+            value: "1"
+          - name: CSI_PROVISIONER_REPLICA_COUNT
+            value: "1"
+          - name: CSI_RESIZER_REPLICA_COUNT
+            value: "1"
+          - name: CSI_SNAPSHOTTER_REPLICA_COUNT
+            value: "1"
+      priorityClassName: "longhorn-critical"
+      serviceAccountName: longhorn-service-account
+      securityContext:
+        runAsUser: 0
+---
+# Source: longhorn/templates/deployment-ui.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+    app: longhorn-ui
+  name: longhorn-ui
+  namespace: longhorn-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: longhorn-ui
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: longhorn
+        helm.sh/chart: longhorn-1.8.0
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/instance: longhorn
+        app.kubernetes.io/version: v1.8.0
+        app: longhorn-ui
+    spec:
+      serviceAccountName: longhorn-ui-service-account
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 1
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app
+                  operator: In
+                  values:
+                  - longhorn-ui
+              topologyKey: kubernetes.io/hostname
+      containers:
+      - name: longhorn-ui
+        image: longhornio/longhorn-ui:v1.8.0
+        imagePullPolicy: IfNotPresent
+        volumeMounts:
+        - name : nginx-cache
+          mountPath: /var/cache/nginx/
+        - name : nginx-config
+          mountPath: /var/config/nginx/
+        - name: var-run
+          mountPath: /var/run/
+        ports:
+        - containerPort: 8000
+          name: http
+        env:
+          - name: LONGHORN_MANAGER_IP
+            value: "http://longhorn-backend:9500"
+          - name: LONGHORN_UI_PORT
+            value: "8000"
+      volumes:
+      - emptyDir: {}
+        name: nginx-cache
+      - emptyDir: {}
+        name: nginx-config
+      - emptyDir: {}
+        name: var-run
+      priorityClassName: "longhorn-critical"
+---
+# Source: longhorn/templates/validate-psp-install.yaml
+#
+---
+# Source: longhorn/templates/postupgrade-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  annotations:
+    "helm.sh/hook": post-upgrade
+    "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation
+  name: longhorn-post-upgrade
+  namespace: longhorn-system
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+spec:
+  activeDeadlineSeconds: 900
+  backoffLimit: 1
+  template:
+    metadata:
+      name: longhorn-post-upgrade
+      labels:
+        app.kubernetes.io/name: longhorn
+        helm.sh/chart: longhorn-1.8.0
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/instance: longhorn
+        app.kubernetes.io/version: v1.8.0
+    spec:
+      containers:
+      - name: longhorn-post-upgrade
+        image: longhornio/longhorn-manager:v1.8.0
+        imagePullPolicy: IfNotPresent
+        command:
+        - longhorn-manager
+        - post-upgrade
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+      restartPolicy: OnFailure
+      priorityClassName: "longhorn-critical"
+      serviceAccountName: longhorn-service-account
+---
+# Source: longhorn/templates/preupgrade-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation,hook-failed
+  name: longhorn-pre-upgrade
+  namespace: longhorn-system
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+spec:
+  activeDeadlineSeconds: 900
+  backoffLimit: 1
+  template:
+    metadata:
+      name: longhorn-pre-upgrade
+      labels:
+        app.kubernetes.io/name: longhorn
+        helm.sh/chart: longhorn-1.8.0
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/instance: longhorn
+        app.kubernetes.io/version: v1.8.0
+    spec:
+      containers:
+      - name: longhorn-pre-upgrade
+        image: longhornio/longhorn-manager:v1.8.0
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          privileged: true
+        command:
+        - longhorn-manager
+        - pre-upgrade
+        volumeMounts:
+        - name: proc
+          mountPath: /host/proc/
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+      volumes:
+      - name: proc
+        hostPath:
+          path: /proc/
+      restartPolicy: OnFailure
+      serviceAccountName: longhorn-service-account
+---
+# Source: longhorn/templates/uninstall-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  annotations:
+    "helm.sh/hook": pre-delete
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+  name: longhorn-uninstall
+  namespace: longhorn-system
+  labels:
+    app.kubernetes.io/name: longhorn
+    helm.sh/chart: longhorn-1.8.0
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.8.0
+spec:
+  activeDeadlineSeconds: 900
+  backoffLimit: 1
+  template:
+    metadata:
+      name: longhorn-uninstall
+      labels:
+        app.kubernetes.io/name: longhorn
+        helm.sh/chart: longhorn-1.8.0
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/instance: longhorn
+        app.kubernetes.io/version: v1.8.0
+    spec:
+      containers:
+      - name: longhorn-uninstall
+        image: longhornio/longhorn-manager:v1.8.0
+        imagePullPolicy: IfNotPresent
+        command:
+        - longhorn-manager
+        - uninstall
+        - --force
+        env:
+        - name: LONGHORN_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+      restartPolicy: Never
+      priorityClassName: "longhorn-critical"
+      serviceAccountName: longhorn-service-account
diff --git a/longhorn-system/minio-backupstore.yaml b/longhorn-system/minio-backupstore.yaml
new file mode 100644
index 0000000..2ff8d59
--- /dev/null
+++ b/longhorn-system/minio-backupstore.yaml
@@ -0,0 +1,91 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: minio-secret
+  namespace: default
+type: Opaque
+data:
+  AWS_ACCESS_KEY_ID: bG9uZ2hvcm4tdGVzdC1hY2Nlc3Mta2V5 # longhorn-test-access-key
+  AWS_SECRET_ACCESS_KEY: bG9uZ2hvcm4tdGVzdC1zZWNyZXQta2V5 # longhorn-test-secret-key
+  AWS_ENDPOINTS: aHR0cHM6Ly9taW5pby1zZXJ2aWNlLmRlZmF1bHQ6OTAwMA== # https://minio-service.default:9000
+  AWS_CERT: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURMRENDQWhTZ0F3SUJBZ0lSQU1kbzQycGhUZXlrMTcvYkxyWjVZRHN3RFFZSktvWklodmNOQVFFTEJRQXcKR2pFWU1CWUdBMVVFQ2hNUFRHOXVaMmh2Y200Z0xTQlVaWE4wTUNBWERUSXdNRFF5TnpJek1EQXhNVm9ZRHpJeApNakF3TkRBek1qTXdNREV4V2pBYU1SZ3dGZ1lEVlFRS0V3OU1iMjVuYUc5eWJpQXRJRlJsYzNRd2dnRWlNQTBHCkNTcUdTSWIzRFFFQkFRVUFBNElCRHdBd2dnRUtBb0lCQVFEWHpVdXJnUFpEZ3pUM0RZdWFlYmdld3Fvd2RlQUQKODRWWWF6ZlN1USs3K21Oa2lpUVBvelVVMmZvUWFGL1BxekJiUW1lZ29hT3l5NVhqM1VFeG1GcmV0eDBaRjVOVgpKTi85ZWFJNWRXRk9teHhpMElPUGI2T0RpbE1qcXVEbUVPSXljdjRTaCsvSWo5Zk1nS0tXUDdJZGxDNUJPeThkCncwOVdkckxxaE9WY3BKamNxYjN6K3hISHd5Q05YeGhoRm9tb2xQVnpJbnlUUEJTZkRuSDBuS0lHUXl2bGhCMGsKVHBHSzYxc2prZnFTK3hpNTlJeHVrbHZIRXNQcjFXblRzYU9oaVh6N3lQSlorcTNBMWZoVzBVa1JaRFlnWnNFbQovZ05KM3JwOFhZdURna2kzZ0UrOElXQWRBWHExeWhqRDdSSkI4VFNJYTV0SGpKUUtqZ0NlSG5HekFnTUJBQUdqCmF6QnBNQTRHQTFVZER3RUIvd1FFQXdJQ3BEQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBVEFQQmdOVkhSTUIKQWY4RUJUQURBUUgvTURFR0ExVWRFUVFxTUNpQ0NXeHZZMkZzYUc5emRJSVZiV2x1YVc4dGMyVnlkbWxqWlM1awpaV1poZFd4MGh3Ui9BQUFCTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFDbUZMMzlNSHVZMzFhMTFEajRwMjVjCnFQRUM0RHZJUWozTk9kU0dWMmQrZjZzZ3pGejFXTDhWcnF2QjFCMVM2cjRKYjJQRXVJQkQ4NFlwVXJIT1JNU2MKd3ViTEppSEtEa0Jmb2U5QWI1cC9VakpyS0tuajM0RGx2c1cvR3AwWTZYc1BWaVdpVWorb1JLbUdWSTI0Q0JIdgpnK0JtVzNDeU5RR1RLajk0eE02czNBV2xHRW95YXFXUGU1eHllVWUzZjFBWkY5N3RDaklKUmVWbENtaENGK0JtCmFUY1RSUWN3cVdvQ3AwYmJZcHlERFlwUmxxOEdQbElFOW8yWjZBc05mTHJVcGFtZ3FYMmtYa2gxa3lzSlEralAKelFadHJSMG1tdHVyM0RuRW0yYmk0TktIQVFIcFc5TXUxNkdRakUxTmJYcVF0VEI4OGpLNzZjdEg5MzRDYWw2VgotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t
+  AWS_CERT_KEY: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRRFh6VXVyZ1BaRGd6VDMKRFl1YWViZ2V3cW93ZGVBRDg0VllhemZTdVErNyttTmtpaVFQb3pVVTJmb1FhRi9QcXpCYlFtZWdvYU95eTVYagozVUV4bUZyZXR4MFpGNU5WSk4vOWVhSTVkV0ZPbXh4aTBJT1BiNk9EaWxNanF1RG1FT0l5Y3Y0U2grL0lqOWZNCmdLS1dQN0lkbEM1Qk95OGR3MDlXZHJMcWhPVmNwSmpjcWIzeit4SEh3eUNOWHhoaEZvbW9sUFZ6SW55VFBCU2YKRG5IMG5LSUdReXZsaEIwa1RwR0s2MXNqa2ZxUyt4aTU5SXh1a2x2SEVzUHIxV25Uc2FPaGlYejd5UEpaK3EzQQoxZmhXMFVrUlpEWWdac0VtL2dOSjNycDhYWXVEZ2tpM2dFKzhJV0FkQVhxMXloakQ3UkpCOFRTSWE1dEhqSlFLCmpnQ2VIbkd6QWdNQkFBRUNnZ0VBZlVyQ1hrYTN0Q2JmZjNpcnp2cFFmZnVEbURNMzV0TmlYaDJTQVpSVW9FMFYKbSsvZ1UvdnIrN2s2eUgvdzhMOXhpZXFhQTljVkZkL0JuTlIrMzI2WGc2dEpCNko2ZGZxODJZdmZOZ0VDaUFMaQpqalNGemFlQmhnT3ZsWXZHbTR5OTU1Q0FGdjQ1cDNac1VsMTFDRXJlL1BGbGtaWHRHeGlrWFl6NC85UTgzblhZCnM2eDdPYTgyUjdwT2lraWh3Q0FvVTU3Rjc4ZWFKOG1xTmkwRlF2bHlxSk9QMTFCbVp4dm54ZU11S2poQjlPTnAKTFNwMWpzZXk5bDZNR2pVbjBGTG53RHZkVWRiK0ZlUEkxTjdWYUNBd3hJK3JHa3JTWkhnekhWWE92VUpON2t2QQpqNUZPNW9uNGgvK3hXbkYzM3lxZ0VvWWZ0MFFJL2pXS2NOV1d1a2pCd1FLQmdRRGVFNlJGRUpsT2Q1aVcxeW1qCm45RENnczVFbXFtRXN3WU95bkN3U2RhK1lNNnZVYmlac1k4WW9wMVRmVWN4cUh2NkFQWGpVd2NBUG1QVE9KRW8KMlJtS0xTYkhsTnc4bFNOMWJsWDBEL3Mzamc1R3VlVW9nbW5TVnhMa0h1OFhKR0o3VzFReEUzZG9IUHRrcTNpagpoa09QTnJpZFM0UmxqNTJwYkhscjUvQzRjUUtCZ1FENHhFYmpuck1heFV2b0xxVTRvT2xiOVc5UytSUllTc0cxCmxJUmgzNzZTV0ZuTTlSdGoyMTI0M1hkaE4zUFBtSTNNeiswYjdyMnZSUi9LMS9Cc1JUQnlrTi9kbkVuNVUxQkEKYm90cGZIS1Jvc1FUR1hIQkEvM0JrNC9qOWplU3RmVXgzZ2x3eUI0L2hORy9KM1ZVV2FXeURTRm5qZFEvcGJsRwp6VWlsSVBmK1l3S0JnUUNwMkdYYmVJMTN5TnBJQ3psS2JqRlFncEJWUWVDQ29CVHkvUHRncUtoM3BEeVBNN1kyCnZla09VMWgyQVN1UkhDWHRtQXgzRndvVXNxTFFhY1FEZEw4bXdjK1Y5eERWdU02TXdwMDBjNENVQmE1L2d5OXoKWXdLaUgzeFFRaVJrRTZ6S1laZ3JqSkxYYXNzT1BHS2cxbEFYV1NlckRaV3R3MEEyMHNLdXQ0NlEwUUtCZ0hGZQpxZHZVR0ZXcjhvTDJ0dzlPcmVyZHVJVTh4RnZVZmVFdHRRTVJ2N3pjRE5qT0gxUnJ4Wk9aUW0ySW92dkp6MTIyCnFKMWhPUXJtV3EzTHFXTCtTU3o4L3pqMG4vWERWVUIzNElzTFR2ODJDVnVXN2ZPRHlTSnVDRlpnZ0VVWkxZd3oKWDJRSm4xZGRSV1Z6S3hKczVJbDNXSERqL3dXZWxnaEJSOGtSZEZOM0FvR0FJNldDdjJQQ1lUS1ZZNjAwOFYwbgpyTDQ3YTlPanZ0Yy81S2ZxSjFpMkpKTUgyQi9jbU1WRSs4M2dpODFIU1FqMWErNnBjektmQVppZWcwRk9nL015ClB6VlZRYmpKTnY0QzM5KzdxSDg1WGdZTXZhcTJ0aDFEZWUvQ3NsMlM4QlV0cW5mc0VuMUYwcWhlWUJZb2RibHAKV3RUaE5oRi9oRVhzbkJROURyWkJKT1U9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K
+---
+# same secret for longhorn-system namespace
+apiVersion: v1
+kind: Secret
+metadata:
+  name: minio-secret
+  namespace: longhorn-system
+type: Opaque
+data:
+  AWS_ACCESS_KEY_ID: bG9uZ2hvcm4tdGVzdC1hY2Nlc3Mta2V5 # longhorn-test-access-key
+  AWS_SECRET_ACCESS_KEY: bG9uZ2hvcm4tdGVzdC1zZWNyZXQta2V5 # longhorn-test-secret-key
+  AWS_ENDPOINTS: aHR0cHM6Ly9taW5pby1zZXJ2aWNlLmRlZmF1bHQ6OTAwMA== # https://minio-service.default:9000
+  AWS_CERT: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURMRENDQWhTZ0F3SUJBZ0lSQU1kbzQycGhUZXlrMTcvYkxyWjVZRHN3RFFZSktvWklodmNOQVFFTEJRQXcKR2pFWU1CWUdBMVVFQ2hNUFRHOXVaMmh2Y200Z0xTQlVaWE4wTUNBWERUSXdNRFF5TnpJek1EQXhNVm9ZRHpJeApNakF3TkRBek1qTXdNREV4V2pBYU1SZ3dGZ1lEVlFRS0V3OU1iMjVuYUc5eWJpQXRJRlJsYzNRd2dnRWlNQTBHCkNTcUdTSWIzRFFFQkFRVUFBNElCRHdBd2dnRUtBb0lCQVFEWHpVdXJnUFpEZ3pUM0RZdWFlYmdld3Fvd2RlQUQKODRWWWF6ZlN1USs3K21Oa2lpUVBvelVVMmZvUWFGL1BxekJiUW1lZ29hT3l5NVhqM1VFeG1GcmV0eDBaRjVOVgpKTi85ZWFJNWRXRk9teHhpMElPUGI2T0RpbE1qcXVEbUVPSXljdjRTaCsvSWo5Zk1nS0tXUDdJZGxDNUJPeThkCncwOVdkckxxaE9WY3BKamNxYjN6K3hISHd5Q05YeGhoRm9tb2xQVnpJbnlUUEJTZkRuSDBuS0lHUXl2bGhCMGsKVHBHSzYxc2prZnFTK3hpNTlJeHVrbHZIRXNQcjFXblRzYU9oaVh6N3lQSlorcTNBMWZoVzBVa1JaRFlnWnNFbQovZ05KM3JwOFhZdURna2kzZ0UrOElXQWRBWHExeWhqRDdSSkI4VFNJYTV0SGpKUUtqZ0NlSG5HekFnTUJBQUdqCmF6QnBNQTRHQTFVZER3RUIvd1FFQXdJQ3BEQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBVEFQQmdOVkhSTUIKQWY4RUJUQURBUUgvTURFR0ExVWRFUVFxTUNpQ0NXeHZZMkZzYUc5emRJSVZiV2x1YVc4dGMyVnlkbWxqWlM1awpaV1poZFd4MGh3Ui9BQUFCTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFDbUZMMzlNSHVZMzFhMTFEajRwMjVjCnFQRUM0RHZJUWozTk9kU0dWMmQrZjZzZ3pGejFXTDhWcnF2QjFCMVM2cjRKYjJQRXVJQkQ4NFlwVXJIT1JNU2MKd3ViTEppSEtEa0Jmb2U5QWI1cC9VakpyS0tuajM0RGx2c1cvR3AwWTZYc1BWaVdpVWorb1JLbUdWSTI0Q0JIdgpnK0JtVzNDeU5RR1RLajk0eE02czNBV2xHRW95YXFXUGU1eHllVWUzZjFBWkY5N3RDaklKUmVWbENtaENGK0JtCmFUY1RSUWN3cVdvQ3AwYmJZcHlERFlwUmxxOEdQbElFOW8yWjZBc05mTHJVcGFtZ3FYMmtYa2gxa3lzSlEralAKelFadHJSMG1tdHVyM0RuRW0yYmk0TktIQVFIcFc5TXUxNkdRakUxTmJYcVF0VEI4OGpLNzZjdEg5MzRDYWw2VgotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: longhorn-test-minio
+  namespace: default
+  labels:
+    app: longhorn-test-minio
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: longhorn-test-minio
+  template:
+    metadata:
+      labels:
+        app: longhorn-test-minio
+    spec:
+      volumes:
+      - name: minio-volume
+        emptyDir: {}
+      - name: minio-certificates
+        secret:
+          secretName: minio-secret
+          items:
+          - key: AWS_CERT
+            path: public.crt
+          - key: AWS_CERT_KEY
+            path: private.key
+      containers:
+      - name: minio
+        image: minio/minio:RELEASE.2022-02-01T18-00-14Z
+        command: ["sh", "-c", "mkdir -p /storage/backupbucket && mkdir -p /root/.minio/certs && ln -s /root/certs/private.key /root/.minio/certs/private.key && ln -s /root/certs/public.crt /root/.minio/certs/public.crt && exec minio server /storage"]
+        env:
+        - name: MINIO_ROOT_USER
+          valueFrom:
+            secretKeyRef:
+              name: minio-secret
+              key: AWS_ACCESS_KEY_ID
+        - name: MINIO_ROOT_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: minio-secret
+              key: AWS_SECRET_ACCESS_KEY
+        ports:
+        - containerPort: 9000
+        volumeMounts:
+        - name: minio-volume
+          mountPath: "/storage"
+        - name: minio-certificates
+          mountPath: "/root/certs"
+          readOnly: true
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio-service
+  namespace: default
+spec:
+  selector:
+    app: longhorn-test-minio
+  ports:
+    - port: 9000
+      targetPort: 9000
+      protocol: TCP
+  sessionAffinity: ClientIP
diff --git a/metrics/grafana-configmap.yaml b/metrics/grafana-configmap.yaml
new file mode 100644
index 0000000..48237a3
--- /dev/null
+++ b/metrics/grafana-configmap.yaml
@@ -0,0 +1,60 @@
+apiVersion: v1
+metadata:
+  name: grafana-config
+  namespace: metrics
+  labels:
+    app.kubernetes.io/name: grafana
+  annotations:
+    use-subpath: "true"
+data:
+  datasources.yaml: |
+    apiVersion: 1
+    datasources:
+      - name: Tempo
+        uid: tempo
+        orgId: 1
+        type: tempo
+        access: proxy
+        url: "http://tempo:3200"
+        jsonData:
+          tracesToLogsV2:
+            datasourceUid: loki
+          tracesToProfiles:
+            datasourceUid: pyroscope
+            profileTypeId: "process_cpu:cpu:nanoseconds:cpu:nanoseconds"
+      - name: Loki
+        uid: loki
+        type: loki
+        access: proxy
+        orgId: 1
+        url: http://loki:3100
+        version: 1
+        editable: false
+        jsonData:
+          timeout: 60
+          maxLines: 1000
+      - name: Prometheus
+        uid: prometheus
+        type: prometheus
+        access: proxy
+        url: http://prometheus-server:9090
+        editable: false
+        jsonData:
+          httpMethod: "POST"
+          prometheusType: "Prometheus"
+          prometheusVersion: "2.9.1"
+          tlsSkipVerify: true
+          timeout: 30
+      - name: Pyroscope
+        uid: pyroscope
+        type: grafana-pyroscope-datasource
+        access: proxy
+        orgId: 1
+        url: http://pyroscope:4040
+        editable: false
+
+    deleteDatasources:
+      - name: "Prometheus"
+        uid: prometheus
+        orgId: 1
+kind: ConfigMap
diff --git a/metrics/grafana-data-persistentvolumeclaim.yaml b/metrics/grafana-data-persistentvolumeclaim.yaml
new file mode 100644
index 0000000..2ab8164
--- /dev/null
+++ b/metrics/grafana-data-persistentvolumeclaim.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: grafana-data
+  namespace: metrics
+  labels:
+    app.kubernetes.io/name: grafana
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 100Mi
diff --git a/metrics/grafana-deployment.yaml b/metrics/grafana-deployment.yaml
new file mode 100644
index 0000000..dcfb505
--- /dev/null
+++ b/metrics/grafana-deployment.yaml
@@ -0,0 +1,84 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grafana
+  namespace: metrics
+  labels:
+    app.kubernetes.io/name: grafana
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: grafana
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      namespace: metrics
+      labels:
+        app.kubernetes.io/name: grafana
+    spec:
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+      containers:
+        - name: grafana
+          image: grafana/grafana:11.1.0
+          env:
+            - name: GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP
+              value: "true"
+            - name: GF_AUTH_GENERIC_OAUTH_API_URL
+              value: https://sso.konfach.ru/realms/konfach/protocol/openid-connect/userinfo
+            - name: GF_AUTH_GENERIC_OAUTH_AUTH_URL
+              value: https://sso.konfach.ru/realms/konfach/protocol/openid-connect/auth
+            - name: GF_AUTH_GENERIC_OAUTH_CLIENT_ID
+              value: grafana
+            - name: GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET
+              value: oraMLSBuIaSPqZElSNRZ6gntM2xizjXL
+            - name: GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH
+              value: email
+            - name: GF_AUTH_GENERIC_OAUTH_ENABLED
+              value: "true"
+            - name: GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH
+              value: username
+            - name: GF_AUTH_GENERIC_OAUTH_NAME
+              value: KonfachSSO
+            - name: GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH
+              value: full_name
+            - name: GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH
+              value: contains(realm_access.roles[*], 'developer') && 'Editor'
+            - name: GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_STRICT
+              value: "true"
+            - name: GF_AUTH_GENERIC_OAUTH_SCOPES
+              value: openid email profile offline_access roles
+            - name: GF_AUTH_GENERIC_OAUTH_TOKEN_URL
+              value: https://sso.konfach.ru/realms/konfach/protocol/openid-connect/token
+            - name: GF_SECURITY_ADMIN_PASSWORD
+            - name: GF_SERVER_ROOT_URL
+            - name: GF_SERVER_SERVE_FROM_SUB_PATH
+              value: "false"
+          resources:
+            limits:
+              memory: "512Mi"
+              cpu: "500m"
+            requests:
+              memory: "256Mi"
+              cpu: "250m"
+          volumeMounts:
+            - mountPath: /var/lib/grafana
+              name: grafana-data
+            - mountPath: /etc/grafana/provisioning/datasources/ds.yaml
+              name: grafana-config
+              subPath: ds.yaml
+      restartPolicy: Always
+      volumes:
+        - name: grafana-data
+          persistentVolumeClaim:
+            claimName: grafana-data
+        - name: grafana-config
+          configMap:
+            name: grafana-config
+            items:
+              - key: datasources.yaml
+                path: ds.yaml
diff --git a/metrics/loki-configmap.yaml b/metrics/loki-configmap.yaml
new file mode 100644
index 0000000..c79a0dd
--- /dev/null
+++ b/metrics/loki-configmap.yaml
@@ -0,0 +1,73 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: loki-config
+  namespace: metrics
+  annotations:
+    use-subpath: "true"
+  labels:
+    app.kubernetes.io/name: loki
+data:
+  config.yaml: |
+    auth_enabled: false
+
+    server:
+      http_listen_port: 3100
+
+    common:
+      instance_addr: 127.0.0.1
+      path_prefix: /loki
+      storage:
+        filesystem:
+          chunks_directory: /loki/chunks
+          rules_directory: /loki/rules
+      replication_factor: 1
+      ring:
+        kvstore:
+          store: inmemory
+
+    limits_config:
+      allow_structured_metadata: true
+      retention_period: 90d
+      otlp_config:
+        resource_attributes:
+          attributes_config:
+            - action: index_label
+              attributes:
+                - component
+                - function
+                - endpoint
+
+    query_range:
+      results_cache:
+        cache:
+          embedded_cache:
+            enabled: true
+
+    compactor:
+      working_directory: /loki/compactor
+      compaction_interval: 10m
+      retention_enabled: true
+      retention_delete_delay: 2h
+      retention_delete_worker_count: 150
+      delete_request_store: filesystem
+
+    schema_config:
+      configs:
+        - from: 2020-10-24
+          store: boltdb-shipper
+          object_store: filesystem
+          schema: v13
+          index:
+            prefix: index_
+            period: 24h
+        - from: 2024-04-12
+          object_store: filesystem
+          store: tsdb
+          schema: v13
+          index:
+            prefix: index_
+            period: 24h
+
+    ruler:
+      alertmanager_url: http://localhost:9093
diff --git a/metrics/loki-data-persistentvolumeclaim.yaml b/metrics/loki-data-persistentvolumeclaim.yaml
new file mode 100644
index 0000000..7c1b606
--- /dev/null
+++ b/metrics/loki-data-persistentvolumeclaim.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: loki-data
+  namespace: metrics
+  labels:
+    app.kubernetes.io/name: loki
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 100Mi
diff --git a/metrics/loki-deployment.yaml b/metrics/loki-deployment.yaml
new file mode 100644
index 0000000..a2ac210
--- /dev/null
+++ b/metrics/loki-deployment.yaml
@@ -0,0 +1,49 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: loki
+  namespace: metrics
+  labels:
+    app.kubernetes.io/name: loki
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: loki
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      namespace: metrics
+      labels:
+        app.kubernetes.io/name: loki
+    spec:
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+      containers:
+        - args:
+            - -config.file=/etc/loki/config.yaml
+          image: grafana/loki:3.1.0
+          name: loki
+          ports:
+            - containerPort: 3100
+              protocol: TCP
+          volumeMounts:
+            - mountPath: /etc/loki/config.yaml
+              name: loki-config
+              subPath: config.yaml
+            - mountPath: /loki
+              name: loki-data
+      restartPolicy: Always
+      volumes:
+        - configMap:
+            items:
+              - key: config.yaml
+                path: config.yaml
+            name: loki-config
+          name: loki-config
+        - name: loki-data
+          persistentVolumeClaim:
+            claimName: loki-data
diff --git a/metrics/loki-service.yaml b/metrics/loki-service.yaml
new file mode 100644
index 0000000..9e364ba
--- /dev/null
+++ b/metrics/loki-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: loki
+  namespace: metrics
+  labels:
+    app.kubernetes.io/name: loki
+spec:
+  ports:
+    - name: "3100"
+      port: 3100
+      targetPort: 3100
+  selector:
+    app.kubernetes.io/name: loki
diff --git a/metrics/otel-collector-configmap.yaml b/metrics/otel-collector-configmap.yaml
new file mode 100644
index 0000000..32dd4a1
--- /dev/null
+++ b/metrics/otel-collector-configmap.yaml
@@ -0,0 +1,76 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: otel-collector-config
+  namespace: metrics
+  annotations:
+    use-subpath: "true"
+  labels:
+    app.kubernetes.io/name: otel-collector
+data:
+  config.yaml: |
+    extensions:
+      health_check:
+      # pprof:
+      zpages:
+
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+            endpoint: 0.0.0.0:4317
+          http:
+            endpoint: 0.0.0.0:4318
+      prometheus/self:
+        config:
+          scrape_configs:
+            - job_name: otel-collector-metrics
+              scrape_interval: 10s
+              static_configs:
+                - targets: ["localhost:8888"]
+                  labels: { "instance": "bigserver-otel-collector" }
+
+    processors:
+      batch:
+
+    exporters:
+      otlp/tempo:
+        endpoint: tempo:4317
+        tls:
+          insecure: true
+      otlphttp/prometheus-server:
+        endpoint: http://prometheus-server:9090/api/v1/otlp
+      otlphttp/loki:
+        endpoint: http://loki:3100/otlp
+
+    connectors:
+      spanmetrics:
+        namespace: span.metrics
+        histogram:
+          explicit:
+            buckets: [100us, 1ms, 2ms, 6ms, 10ms, 100ms, 250ms, 1s, 5s, 30s, 1m, 5m]
+        exemplars:
+          enabled: true
+
+    service:
+      extensions:
+        - health_check
+        - zpages
+
+      pipelines:
+        traces:
+          receivers: [otlp]
+          processors: [batch]
+          exporters: [otlp/tempo, spanmetrics]
+        metrics:
+          receivers: [otlp, prometheus/self, spanmetrics]
+          processors: [batch]
+          exporters: [otlphttp/prometheus-server]
+        logs:
+          receivers: [otlp]
+          processors: [batch]
+          exporters: [otlphttp/loki]
+      telemetry:
+        metrics:
+          address: 0.0.0.0:8888
+          level: detailed
diff --git a/metrics/otel-collector-deployment.yaml b/metrics/otel-collector-deployment.yaml
new file mode 100644
index 0000000..7b9e6f3
--- /dev/null
+++ b/metrics/otel-collector-deployment.yaml
@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: metrics
+  labels:
+    app.kubernetes.io/name: otel-collector
+  name: otel-collector
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: otel-collector
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      namespace: metrics
+      labels:
+        app.kubernetes.io/name: otel-collector
+    spec:
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+      containers:
+        - args:
+            - --config=/etc/otel-collector-config.yaml
+          image: otel/opentelemetry-collector-contrib:0.104.0
+          name: otel-collector
+          ports:
+            - containerPort: 4317
+              protocol: TCP
+            - containerPort: 4318
+              protocol: TCP
+
+          volumeMounts:
+            - mountPath: /etc/otel-collector-config.yaml
+              name: otel-collector-config
+              subPath: otel-collector-config.yaml
+      restartPolicy: Always
+      volumes:
+        - configMap:
+            items:
+              - key: config.yaml
+                path: otel-collector-config.yaml
+            name: otel-collector-config
+          name: otel-collector-config
diff --git a/metrics/otel-collector-service.yaml b/metrics/otel-collector-service.yaml
new file mode 100644
index 0000000..3f41017
--- /dev/null
+++ b/metrics/otel-collector-service.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  namespace: metrics
+  labels:
+    app.kubernetes.io/name: otel-collector
+  name: otel-collector
+spec:
+  ports:
+    - name: "4317"
+      port: 4317
+      targetPort: 4317
+    - name: "4318"
+      port: 4318
+      targetPort: 4318
+  selector:
+    app.kubernetes.io/name: otel-collector
diff --git a/metrics/prometheus.yaml b/metrics/prometheus.yaml
new file mode 100644
index 0000000..9099e91
--- /dev/null
+++ b/metrics/prometheus.yaml
@@ -0,0 +1,172 @@
+---
+# Source: prometheus/templates/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/component: server
+    app.kubernetes.io/name: prometheus
+    app.kubernetes.io/instance: prometheus
+    app.kubernetes.io/version: v3.2.1
+    app.kubernetes.io/part-of: prometheus
+  name: prometheus-server
+  namespace: metrics
+  annotations: {}
+---
+# Source: prometheus/templates/cm.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-server
+  namespace: metrics
+  labels:
+    app.kubernetes.io/component: server
+    app.kubernetes.io/name: prometheus
+    app.kubernetes.io/instance: prometheus
+    app.kubernetes.io/version: v3.2.1
+    app.kubernetes.io/part-of: prometheus
+data:
+  allow-snippet-annotations: "false"
+  prometheus.yml: |
+    global:
+      evaluation_interval: 1m
+    storage:
+      tsdb:
+        out_of_order_time_window: 30m
+    otlp:
+      translation_strategy: NoUTF8EscapingWithSuffixes
+---
+# Source: prometheus/templates/pvc.yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  labels:
+    app.kubernetes.io/component: server
+    app.kubernetes.io/name: prometheus
+    app.kubernetes.io/instance: prometheus
+    app.kubernetes.io/version: v3.2.1
+    app.kubernetes.io/part-of: prometheus
+  name: prometheus-server
+  namespace: metrics
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: "8Gi"
+---
+# Source: prometheus/templates/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus-server
+  namespace: metrics
+  labels:
+    app.kubernetes.io/component: server
+    app.kubernetes.io/name: prometheus
+    app.kubernetes.io/instance: prometheus
+    app.kubernetes.io/version: v3.2.1
+    app.kubernetes.io/part-of: prometheus
+spec:
+  type: "ClusterIP"
+  sessionAffinity: None
+  ports:
+    - name: http
+      port: 9090
+      protocol: TCP
+      targetPort: 9090
+  selector:
+    app.kubernetes.io/component: server
+    app.kubernetes.io/name: prometheus
+    app.kubernetes.io/instance: prometheus
+---
+# Source: prometheus/templates/deploy.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/component: server
+    app.kubernetes.io/name: prometheus
+    app.kubernetes.io/instance: prometheus
+    app.kubernetes.io/version: v3.2.1
+    app.kubernetes.io/part-of: prometheus
+  name: prometheus-server
+  namespace: metrics
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: server
+      app.kubernetes.io/name: prometheus
+      app.kubernetes.io/instance: prometheus
+  replicas: 1
+  revisionHistoryLimit: 10
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: server
+        app.kubernetes.io/name: prometheus
+        app.kubernetes.io/instance: prometheus
+        app.kubernetes.io/version: v3.2.1
+        app.kubernetes.io/part-of: prometheus
+    spec:
+      enableServiceLinks: true
+      serviceAccountName: prometheus-server
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+      containers:
+        - name: prometheus-server
+          image: "quay.io/prometheus/prometheus:v3.2.1"
+          args:
+            - --storage.tsdb.retention.time=15d
+            - --config.file=/etc/config/prometheus.yml
+            - --storage.tsdb.path=/data
+            - --web.enable-lifecycle
+            - --web.enable-otlp-receiver
+          ports:
+            - containerPort: 9090
+          resources:
+            limits:
+              cpu: "500m"
+              memory: "256Mi"
+            requests:
+              cpu: "250m"
+          readinessProbe:
+            httpGet:
+              path: /-/ready
+              port: 9090
+              scheme: HTTP
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            timeoutSeconds: 4
+            failureThreshold: 3
+            successThreshold: 1
+          livenessProbe:
+            httpGet:
+              path: /-/healthy
+              port: 9090
+              scheme: HTTP
+            initialDelaySeconds: 30
+            periodSeconds: 15
+            timeoutSeconds: 10
+            failureThreshold: 3
+            successThreshold: 1
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/config
+              readOnly: true
+            - name: storage-volume
+              mountPath: /data
+              subPath: ""
+      dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 300
+      volumes:
+        - name: config-volume
+          configMap:
+            name: prometheus-server
+        - name: storage-volume
+          persistentVolumeClaim:
+            claimName: prometheus-server
diff --git a/metrics/pyroscope-configmap.yaml b/metrics/pyroscope-configmap.yaml
new file mode 100644
index 0000000..a402c05
--- /dev/null
+++ b/metrics/pyroscope-configmap.yaml
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: pyroscope-config
+  namespace: metrics
+  annotations:
+    use-subpath: "true"
+  labels:
+    app.kubernetes.io/name: pyroscope
+
+data:
+  config.yaml: |
+    tracing:
+      enabled: true
+      profiling_enabled: true
+
+    pyroscopedb:
+      max_block_duration: 5m
diff --git a/metrics/pyroscope-data-persistentvolumeclaim.yaml b/metrics/pyroscope-data-persistentvolumeclaim.yaml
new file mode 100644
index 0000000..4a0970f
--- /dev/null
+++ b/metrics/pyroscope-data-persistentvolumeclaim.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pyroscope-data
+  namespace: metrics
+  labels:
+    app.kubernetes.io/name: pyroscope
+
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 100Mi
diff --git a/metrics/pyroscope-deployment.yaml b/metrics/pyroscope-deployment.yaml
new file mode 100644
index 0000000..6e826cb
--- /dev/null
+++ b/metrics/pyroscope-deployment.yaml
@@ -0,0 +1,48 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: metrics
+  labels:
+    app.kubernetes.io/name: pyroscope
+  name: pyroscope
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: pyroscope
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      namespace: metrics
+      labels:
+        app.kubernetes.io/name: pyroscope
+    spec:
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+      containers:
+        - image: grafana/pyroscope:1.7.1
+          name: pyroscope
+          ports:
+            - containerPort: 4040
+              protocol: TCP
+
+          volumeMounts:
+            - mountPath: /etc/pyroscope.yml
+              name: pyroscope-config
+              subPath: pyroscope.yml
+            - mountPath: /data
+              name: pyroscope-data
+      restartPolicy: Always
+      volumes:
+        - configMap:
+            items:
+              - key: config.yaml
+                path: pyroscope.yml
+            name: pyroscope-config
+          name: pyroscope-config
+        - name: pyroscope-data
+          persistentVolumeClaim:
+            claimName: pyroscope-data
diff --git a/metrics/pyroscope-service.yaml b/metrics/pyroscope-service.yaml
new file mode 100644
index 0000000..0fa40aa
--- /dev/null
+++ b/metrics/pyroscope-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  namespace: metrics
+  labels:
+    app.kubernetes.io/name: pyroscope
+  name: pyroscope
+spec:
+  ports:
+    - name: "4040"
+      port: 4040
+      targetPort: 4040
+  selector:
+    app.kubernetes.io/name: pyroscope
diff --git a/metrics/tempo-configmap.yaml b/metrics/tempo-configmap.yaml
new file mode 100644
index 0000000..e3a402c
--- /dev/null
+++ b/metrics/tempo-configmap.yaml
@@ -0,0 +1,44 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: tempo-config
+  namespace: metrics
+  annotations:
+    use-subpath: "true"
+  labels:
+    app.kubernetes.io/name: tempo
+data:
+  config.yaml: |
+    stream_over_http_enabled: true
+    server:
+      http_listen_port: 3200
+      log_level: info
+
+    distributor:
+      receivers:
+        otlp:
+          protocols:
+            http:
+            grpc:
+
+    ingester:
+      max_block_bytes: 131072000 # 128mb
+
+    compactor:
+      compaction:
+        block_retention: 672h # 28 days
+        compacted_block_retention: 30m
+        retention_concurrency: 2
+
+    storage:
+      trace:
+        backend: local
+        wal:
+          path: /tempo-data/wal # where to store the the wal locally
+        local:
+          path: /tempo-data/blocks
+
+    overrides:
+      defaults:
+        global:
+          max_bytes_per_trace: 10000000
diff --git a/metrics/tempo-data-persistentvolumeclaim.yaml b/metrics/tempo-data-persistentvolumeclaim.yaml
new file mode 100644
index 0000000..5fe6761
--- /dev/null
+++ b/metrics/tempo-data-persistentvolumeclaim.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  namespace: metrics
+  labels:
+    app.kubernetes.io/name: tempo
+  name: tempo-data
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 100Mi
diff --git a/metrics/tempo-deployment.yaml b/metrics/tempo-deployment.yaml
new file mode 100644
index 0000000..e58e14c
--- /dev/null
+++ b/metrics/tempo-deployment.yaml
@@ -0,0 +1,54 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: metrics
+  labels:
+    app.kubernetes.io/name: tempo
+  name: tempo
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: tempo
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      namespace: metrics
+      labels:
+        app.kubernetes.io/name: tempo
+    spec:
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+      containers:
+        - args:
+            - -config.file=/etc/tempo/config.yaml
+          image: grafana/tempo:2.5.0
+          name: tempo
+          ports:
+            - containerPort: 3200
+              protocol: TCP
+            - containerPort: 4317
+              protocol: TCP
+            - containerPort: 4318
+              protocol: TCP
+
+          volumeMounts:
+            - mountPath: /etc/tempo/config.yaml
+              name: tempo-config
+              subPath: config.yaml
+            - mountPath: /tempo-data
+              name: tempo-data
+      restartPolicy: Always
+      volumes:
+        - configMap:
+            items:
+              - key: config.yaml
+                path: config.yaml
+            name: tempo-config
+          name: tempo-config
+        - name: tempo-data
+          persistentVolumeClaim:
+            claimName: tempo-data
diff --git a/metrics/tempo-service.yaml b/metrics/tempo-service.yaml
new file mode 100644
index 0000000..4dae824
--- /dev/null
+++ b/metrics/tempo-service.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: tempo
+  namespace: metrics
+  labels:
+    app.kubernetes.io/name: tempo
+spec:
+  ports:
+    - name: "3200"
+      port: 3200
+      targetPort: 3200
+    - name: "4317"
+      port: 4317
+      targetPort: 4317
+    - name: "4318"
+      port: 4318
+      targetPort: 4318
+  selector:
+    app.kubernetes.io/name: tempo