diff --git a/docs/persistence.md b/docs/persistence.md new file mode 100644 index 000000000..77788d11e --- /dev/null +++ b/docs/persistence.md @@ -0,0 +1,72 @@ +# Data Persistence + +By default, Warnet nodes use ephemeral storage, meaning all data is lost when a pod is deleted or restarted. This document describes how to enable persistent storage to be able to use warnet for persistent development environment, such that blockchain data, wallet information, and other node state can survive pod restarts and network redeployments. This is done with Kubernetes Persistent Volume Claims (PVCs), which persist independently of pod lifecycle. + +Persistence is available for: +- **Bitcoin Core** nodes +- **LND** nodes +- **CLN** nodes + +## Enabling Persistence + +Persistence is configured per-node in the network graph definition. Add a `persistence` section to any node's configuration. This creates a new PVC for that node, which is then mounted to the appropriate data directory inside the container. + +Also add `restartPolicy: Always` to the node's configuration to ensure that the pod is restarted if it is deleted or crashes. This is important to ensure proper restart after restart of the kubernetes cluster. If there is a risk of hard resets of the cluster, add `reindex=1` to bitcoin_core config to reindex the blockchain on startup and fix potential corrupted chain state. + +### Bitcoin Core Node + +```yaml +bitcoin_core: + image: bitcoincore-27.1:latest + restartPolicy: Always + persistence: + enabled: true + size: 20Gi # optional, default is 20Gi + storageClass: "" # optional, default is cluster default storage class + accessMode: ReadWriteOncePod # optional, default is ReadWriteOncePod. For compatibility with older Kubernetes versions, you may need to set this to ReadWriteOnce + config: | + reindex=1 +``` + +### Lightning Node + +```yaml +: + image: + tag: + restartPolicy: Always + persistence: + enabled: true + size: 10Gi # optional, default is 10Gi + storageClass: "" # optional, default is cluster default storage class + accessMode: ReadWriteOncePod # optional, default is ReadWriteOncePod. For compatibility with older Kubernetes versions, you may need to set this to ReadWriteOnce +``` + +## Existing PVCs + +To use custom made PVC or PVC from previous deployment, use the `existingClaim` field to reference an existing PVC by name. If the network configuration or namespace did not change, there is no need to explicitly set the `existingClaim`. The existing PVC is used by default, since its generated name matches the default pattern. To explicitly use a PVC set the name like this: + +```yaml +persistence: + enabled: true + existingClaim: "tank-0001.default-bitcoincore-data" +``` + +The generated PVC names follow the pattern: +`.--data` + +For example for a bitcoin core node: +`tank-0001.default-bitcoincore-data` + +And for a LND node: +`tank-0001-ln.default-lnd-data` + +Get the list of PVCs in the cluster with `kubectl get pvc -A` and delete any PVCs that are no longer needed with `kubectl delete pvc -n `. + +## Mount Paths + +When persistence is enabled, the following directories are persisted in the PVCs: + +- **Bitcoin Core:** `/root/.bitcoin/` +- **LND:** `/root/.lnd/` +- **CLN:** `/root/.lightning/` \ No newline at end of file diff --git a/resources/charts/bitcoincore/charts/cln/templates/pod.yaml b/resources/charts/bitcoincore/charts/cln/templates/pod.yaml index 59d212d0e..f96cb40a9 100644 --- a/resources/charts/bitcoincore/charts/cln/templates/pod.yaml +++ b/resources/charts/bitcoincore/charts/cln/templates/pod.yaml @@ -56,6 +56,8 @@ spec: - mountPath: /root/.lightning/config name: config subPath: config + - mountPath: /root/.lightning + name: cln-data {{- with .Values.extraContainers }} {{- toYaml . | nindent 4 }} {{- end }} @@ -80,6 +82,18 @@ spec: - configMap: name: {{ include "cln.fullname" . }} name: config + - name: cln-data + {{- if .Values.persistence.enabled }} + {{- if .Values.persistence.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.persistence.existingClaim }} + {{- else }} + persistentVolumeClaim: + claimName: {{ include "cln.fullname" . }}.{{ .Release.Namespace }}-cln-data + {{- end }} + {{- else }} + emptyDir: {} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 4 }} diff --git a/resources/charts/bitcoincore/charts/cln/templates/pvc.yaml b/resources/charts/bitcoincore/charts/cln/templates/pvc.yaml new file mode 100644 index 000000000..0dce96d43 --- /dev/null +++ b/resources/charts/bitcoincore/charts/cln/templates/pvc.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "cln.fullname" . }}.{{ .Release.Namespace }}-cln-data + labels: + {{- include "cln.labels" . | nindent 4 }} + annotations: + "helm.sh/resource-policy": keep +spec: + accessModes: + - {{ .Values.persistence.accessMode }} + resources: + requests: + storage: {{ .Values.persistence.size }} + {{- if .Values.persistence.storageClass }} + storageClassName: {{ .Values.persistence.storageClass }} + {{- end }} +{{- end }} diff --git a/resources/charts/bitcoincore/charts/cln/values.yaml b/resources/charts/bitcoincore/charts/cln/values.yaml index eaae7d2a3..953280445 100644 --- a/resources/charts/bitcoincore/charts/cln/values.yaml +++ b/resources/charts/bitcoincore/charts/cln/values.yaml @@ -82,6 +82,15 @@ startupProbe: - "-c" - "lightning-cli createrune > /working/rune.json" +# Node data persistence configuration. Create persistent volume claim, or use an existing one. +persistence: + enabled: false + storageClass: "" + accessMode: ReadWriteOncePod + size: 10Gi + # Use existing persistent volume claim instead of creating a new one. + existingClaim: "" + # Additional volumes on the output Deployment definition. volumes: - name: working diff --git a/resources/charts/bitcoincore/charts/lnd/templates/pod.yaml b/resources/charts/bitcoincore/charts/lnd/templates/pod.yaml index 1b0305805..7c2162159 100644 --- a/resources/charts/bitcoincore/charts/lnd/templates/pod.yaml +++ b/resources/charts/bitcoincore/charts/lnd/templates/pod.yaml @@ -79,7 +79,7 @@ spec: - mountPath: /root/.lnd/tls.cert name: config subPath: tls.cert - - name: shared-volume + - name: lnd-data mountPath: /root/.lnd/ {{- with .Values.extraContainers }} {{- toYaml . | nindent 4 }} @@ -95,7 +95,7 @@ spec: - "--macaroonpath=/root/.lnd/data/chain/bitcoin/{{ .Values.global.chain }}/admin.macaroon" - "--httplisten=0.0.0.0:{{ .Values.circuitbreaker.httpPort }}" volumeMounts: - - name: shared-volume + - name: lnd-data mountPath: /root/.lnd/ - name: config mountPath: /tls.cert @@ -108,8 +108,18 @@ spec: - configMap: name: {{ include "lnd.fullname" . }} name: config - - name: shared-volume + - name: lnd-data + {{- if .Values.persistence.enabled }} + {{- if .Values.persistence.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.persistence.existingClaim }} + {{- else }} + persistentVolumeClaim: + claimName: {{ include "lnd.fullname" . }}.{{ .Release.Namespace }}-lnd-data + {{- end }} + {{- else }} emptyDir: {} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 4 }} diff --git a/resources/charts/bitcoincore/charts/lnd/templates/pvc.yaml b/resources/charts/bitcoincore/charts/lnd/templates/pvc.yaml new file mode 100644 index 000000000..ab9f28c2a --- /dev/null +++ b/resources/charts/bitcoincore/charts/lnd/templates/pvc.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "lnd.fullname" . }}.{{ .Release.Namespace }}-lnd-data + labels: + {{- include "lnd.labels" . | nindent 4 }} + annotations: + "helm.sh/resource-policy": keep +spec: + accessModes: + - {{ .Values.persistence.accessMode }} + resources: + requests: + storage: {{ .Values.persistence.size }} + {{- if .Values.persistence.storageClass }} + storageClassName: {{ .Values.persistence.storageClass }} + {{- end }} +{{- end }} diff --git a/resources/charts/bitcoincore/charts/lnd/values.yaml b/resources/charts/bitcoincore/charts/lnd/values.yaml index 28db1eb86..5a1b7e97f 100644 --- a/resources/charts/bitcoincore/charts/lnd/values.yaml +++ b/resources/charts/bitcoincore/charts/lnd/values.yaml @@ -15,7 +15,7 @@ imagePullSecrets: [] nameOverride: "" fullnameOverride: "" -podLabels: +podLabels: app: "warnet" mission: "lightning" @@ -65,12 +65,11 @@ resources: {} # cpu: 100m # memory: 128Mi - livenessProbe: exec: command: - - pidof - - lnd + - pidof + - lnd failureThreshold: 3 initialDelaySeconds: 60 periodSeconds: 5 @@ -85,6 +84,15 @@ readinessProbe: port: 10009 timeoutSeconds: 1 +# Node data persistence configuration. Create persistent volume claim, or use an existing one. +persistence: + enabled: false + storageClass: "" + accessMode: ReadWriteOncePod + size: 10Gi + # Use existing persistent volume claim instead of creating a new one. + existingClaim: "" + # Additional volumes on the output Deployment definition. volumes: [] # - name: foo @@ -128,6 +136,6 @@ defaultConfig: "" channels: [] circuitbreaker: - enabled: false # Default to disabled + enabled: false # Default to disabled image: carlakirkcohen/circuitbreaker:attackathon-test - httpPort: 9235 \ No newline at end of file + httpPort: 9235 diff --git a/resources/charts/bitcoincore/templates/pod.yaml b/resources/charts/bitcoincore/templates/pod.yaml index f8b8d1de1..2577468f9 100644 --- a/resources/charts/bitcoincore/templates/pod.yaml +++ b/resources/charts/bitcoincore/templates/pod.yaml @@ -109,7 +109,17 @@ spec: {{- toYaml . | nindent 4 }} {{- end }} - name: data + {{- if .Values.persistence.enabled }} + {{- if .Values.persistence.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.persistence.existingClaim }} + {{- else }} + persistentVolumeClaim: + claimName: {{ include "bitcoincore.fullname" . }}.{{ .Release.Namespace }}-bitcoincore-data + {{- end }} + {{- else }} emptyDir: {} + {{- end }} - name: config configMap: name: {{ include "bitcoincore.fullname" . }} diff --git a/resources/charts/bitcoincore/templates/pvc.yaml b/resources/charts/bitcoincore/templates/pvc.yaml new file mode 100644 index 000000000..ca8510df2 --- /dev/null +++ b/resources/charts/bitcoincore/templates/pvc.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "bitcoincore.fullname" . }}.{{ .Release.Namespace }}-bitcoincore-data + labels: + {{- include "bitcoincore.labels" . | nindent 4 }} + annotations: + "helm.sh/resource-policy": keep +spec: + accessModes: + - {{ .Values.persistence.accessMode }} + resources: + requests: + storage: {{ .Values.persistence.size }} + {{- if .Values.persistence.storageClass }} + storageClassName: {{ .Values.persistence.storageClass }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/resources/charts/bitcoincore/values.yaml b/resources/charts/bitcoincore/values.yaml index 67f1b9b10..92416f6bb 100644 --- a/resources/charts/bitcoincore/values.yaml +++ b/resources/charts/bitcoincore/values.yaml @@ -15,7 +15,7 @@ imagePullSecrets: [] nameOverride: "" fullnameOverride: "" -podLabels: +podLabels: app: "warnet" mission: "tank" @@ -61,12 +61,11 @@ resources: {} # cpu: 100m # memory: 128Mi - livenessProbe: exec: command: - - pidof - - bitcoind + - pidof + - bitcoind failureThreshold: 12 initialDelaySeconds: 5 periodSeconds: 5 @@ -78,6 +77,14 @@ readinessProbe: successThreshold: 1 timeoutSeconds: 10 +# Node data persistence configuration. Create persisten volume claim, or use an existing one. +persistence: + enabled: false + storageClass: "" + accessMode: ReadWriteOncePod + size: 20Gi + # Use existing persistent volume claim instead of creating a new one. + existingClaim: "" # Additional volumes on the output Deployment definition. volumes: [] diff --git a/resources/scenarios/commander.py b/resources/scenarios/commander.py index a6cb541d1..a5129dbd9 100644 --- a/resources/scenarios/commander.py +++ b/resources/scenarios/commander.py @@ -173,8 +173,13 @@ def run_test(self): @staticmethod def ensure_miner(node): wallets = node.listwallets() + if "miner" not in wallets: - node.createwallet("miner", descriptors=True) + allwallets = node.listwalletdir() + if "'miner'" in str(allwallets): + node.loadwallet("miner") + else: + node.createwallet("miner", descriptors=True) return node.get_wallet_rpc("miner") @staticmethod diff --git a/src/warnet/control.py b/src/warnet/control.py index d3183c73a..fdf019172 100644 --- a/src/warnet/control.py +++ b/src/warnet/control.py @@ -33,6 +33,7 @@ get_default_namespace_or, get_mission, get_namespaces, + get_persistent_volume_claims, get_pod, get_pods, pod_log, @@ -161,12 +162,18 @@ def delete_pod(pod_name, namespace): subprocess.Popen(cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) return f"Initiated deletion of pod: {pod_name} in namespace {namespace}" + def delete_persistent_volume_claim(pvc_name, namespace): + cmd = f"kubectl delete pvc --ignore-not-found=true {pvc_name} -n {namespace}" + subprocess.Popen(cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + return f"Initiated deletion of PVC: {pvc_name} in namespace {namespace}" + if not can_delete_pods(): click.secho("You do not have permission to bring down the network.", fg="red") return namespaces = get_namespaces() release_list: list[dict[str, str]] = [] + pvc_list: list[dict[str, str]] = [] for v1namespace in namespaces: namespace = v1namespace.metadata.name command = f"helm list --namespace {namespace} -o json" @@ -176,6 +183,10 @@ def delete_pod(pod_name, namespace): for release in releases: release_list.append({"namespace": namespace, "name": release["name"]}) + pvcs = get_persistent_volume_claims(namespace) + for pvc in pvcs: + pvc_list.append({"namespace": namespace, "name": pvc.metadata.name}) + confirmed = "confirmed" click.secho("Preparing to bring down the running Warnet...", fg="yellow") @@ -207,6 +218,36 @@ def delete_pod(pod_name, namespace): click.secho("Operation cancelled by user", fg="yellow") sys.exit(0) + # Prompt to also delete PVCs containing persistent data + pvc_answers = None + if len(pvc_list) > 0: + table = Table( + title="Persistent Volume Claims to be destroyed", + show_header=True, + header_style="bold red", + ) + table.add_column("Namespace", style="red") + table.add_column("Name", style="red") + for pvc in pvc_list: + table.add_row(pvc["namespace"], pvc["name"]) + console.print(table) + pvc_confirmed = "pvc_confirmed" + pvc_answers = inquirer.prompt( + [ + inquirer.Confirm( + pvc_confirmed, + message=click.style( + "Do you also want to delete all PVCs containing persistent node data?", + fg="yellow", + bold=False, + ), + default=False, + ), + ] + ) + + delete_pvcs = pvc_answers and pvc_answers[pvc_confirmed] + with ThreadPoolExecutor(max_workers=10) as executor: futures = [] @@ -228,6 +269,13 @@ def delete_pod(pod_name, namespace): for pod in pods: futures.append(executor.submit(delete_pod, pod.metadata.name, pod.metadata.namespace)) + # Delete PVCs if confirmed + if delete_pvcs: + for pvc in pvc_list: + futures.append( + executor.submit(delete_persistent_volume_claim, pvc["name"], pvc["namespace"]) + ) + # Wait for all tasks to complete and print results for future in as_completed(futures): console.print(f"[yellow]{future.result()}[/yellow]") diff --git a/src/warnet/k8s.py b/src/warnet/k8s.py index 9ca39feaa..615b65cb3 100644 --- a/src/warnet/k8s.py +++ b/src/warnet/k8s.py @@ -99,6 +99,13 @@ def get_channels(namespace: Optional[str] = None) -> any: return channels +def get_persistent_volume_claims(namespace: Optional[str] = None) -> any: + namespace = get_default_namespace_or(namespace) + sclient = get_static_client() + pvcs = sclient.list_namespaced_persistent_volume_claim(namespace=namespace) + return pvcs.items + + def create_kubernetes_object( kind: str, metadata: dict[str, any], spec: dict[str, any] = None ) -> dict[str, any]: diff --git a/test/test_base.py b/test/test_base.py index 720519b0e..5c5e3fb54 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -52,7 +52,20 @@ def cleanup(self, signum=None, frame=None): session.logfile = sys.stdout session.expect("Do you want to bring down the running Warnet?", timeout=30) session.sendline("y") - session.expect("Warnet teardown process completed", timeout=300) + + # Handle PVC question if asked within 5 seconds + outcome = session.expect( + ["Do you also want to delete all PVCs", pexpect.EOF, pexpect.TIMEOUT], timeout=5 + ) + if outcome == 1: + assert "Warnet teardown process completed" in session.before + self.log.info("Warnet teardown process completed") + else: + if outcome == 0: + session.sendline("y") + else: + self.log.info("Didn't get delete PVC question, continuing...") + session.expect("Warnet teardown process completed", timeout=300) self.wait_for_all_tanks_status(target="stopped", timeout=60, interval=1) except Exception as e: self.log.error(f"Error bringing network down: {e}")