Your IP : 216.73.216.43

Current Path : /proc/thread-self/root/snap/lxd/38450/share/lxd-documentation/howto/cluster_recover/
Upload File :
Current File : //proc/thread-self/root/snap/lxd/38450/share/lxd-documentation/howto/cluster_recover/index.html

<!doctype html>
<html class="no-js" lang="en" data-content_root="../../">
  <head><meta charset="utf-8">
    <meta name="viewport" content="width=device-width,initial-scale=1">
    <meta name="color-scheme" content="light dark"><meta name="viewport" content="width=device-width, initial-scale=1" />
<meta property="og:title" content="How to recover a cluster" />
<meta property="og:type" content="website" />
<meta property="og:url" content="/howto/cluster_recover/" />
<meta property="og:site_name" content="LXD documentation" />
<meta property="og:description" content="It might happen that one or several members of your cluster go offline or become unreachable. If too many cluster members go offline, no operations will be possible on the cluster. See Offline memb..." />
<meta property="og:image" content="https://documentation.ubuntu.com/lxd/latest/_static/lxd_tag.png" />
<meta property="og:image:alt" content="LXD documentation" />
<meta name="description" content="It might happen that one or several members of your cluster go offline or become unreachable. If too many cluster members go offline, no operations will be possible on the cluster. See Offline memb..." />

    <meta property="article:modified_time" content="2024-12-09T15:51:19+00:00" /><link rel="index" title="Index" href="../../genindex/"><link rel="search" title="Search" href="../../search/"><link rel="next" title="How to set up a highly available virtual IP for clusters" href="../cluster_vip/"><link rel="prev" title="How to use placement groups" href="../cluster_placement_groups/">
        <link rel="canonical" href="/howto/cluster_recover/">

    <link rel="shortcut icon" href="../../_static/favicon.ico"><!-- Generated with Sphinx 7.4.7 and Furo 2025.12.19 -->
        <title>How to recover a cluster - LXD documentation</title>
      <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=d111a655" />
    <link rel="stylesheet" type="text/css" href="../../_static/styles/furo.css?v=7bdb33bb" />
    <link rel="stylesheet" type="text/css" href="../../_static/copybutton.css?v=76b2166b" />
    <link rel="stylesheet" type="text/css" href="../../_static/config-options.css" />
    <link rel="stylesheet" type="text/css" href="../../_static/related-links.css" />
    <link rel="stylesheet" type="text/css" href="../../_static/terminal.css" />
    <link rel="stylesheet" type="text/css" href="../../_static/youtube.css" />
    <link rel="stylesheet" type="text/css" href="../../_static/sphinx-design.min.css?v=95c83b7e" />
    <link rel="stylesheet" type="text/css" href="../../_static/styles/furo-extensions.css?v=8dab3a3b" />
    <link rel="stylesheet" type="text/css" href="../../_static/lxd_custom.css?v=bfbf4da2" />
    <link rel="stylesheet" type="text/css" href="../../_static/cookie-banner.css?v=b74831ab" />
    <link rel="stylesheet" type="text/css" href="../../_static/custom.css?v=e189117a" />
    <link rel="stylesheet" type="text/css" href="../../_static/header.css?v=a8078839" />
    <link rel="stylesheet" type="text/css" href="../../_static/github_issue_links.css?v=3d761185" />
    <link rel="stylesheet" type="text/css" href="../../_static/furo_colors.css?v=825fec6f" />
    
</head>
  <body>
    <header id="header" class="p-navigation">
  <!-- Google Tag Manager -->
  <script>
    (function(w, d, s, l, i) {
      w[l] = w[l] || [];
      w[l].push({
        'gtm.start': new Date().getTime(),
        event: 'gtm.js'
      });
      var f = d.getElementsByTagName(s)[0];
      var j = d.createElement(s);
      var dl = '';
      if (l != 'dataLayer') {
          dl = '&l=' + l;
      }
      j.async = true;
      j.src = 'https://www.googletagmanager.com/gtm.js?id=' + i + dl;
      f.parentNode.insertBefore(j, f);
    })(window, document, 'script', 'dataLayer', 'GTM-KNX3CJC');
  </script>
  
  <div class="p-navigation__nav" role="menubar">

    <ul class="p-navigation__links" role="menu">

      <li>
        
        <a class="p-logo" href="https://canonical.com/lxd" aria-current="page">
          <img src="../../_static/lxd_tag.png" alt="Logo" class="p-logo-image">
          <div class="p-logo-text p-heading--4">LXD
          </div>
        </a>
        
      </li>

      <li class="nav-ubuntu-com">
        
        <a href="https://canonical.com/lxd" class="p-navigation__link">canonical.com/lxd</a>
        
      </li>

      <li class="nav-dropdown">
        <a href="#" class="p-navigation__link nav-more-links"
           id="more-resources-toggle"
           aria-haspopup="true"
           aria-expanded="false">
          More resources
        </a>
        <ul class="more-links-dropdown" aria-labelledby="more-resources-toggle">
          
          <li>
            <a href="https://discourse.ubuntu.com/c/lxd/" class="p-navigation__sub-link p-dropdown__link">Discourse</a>
          </li>
          
      
          
      
          
          <li>
            <a href="https://matrix.to/#/#documentation:ubuntu.com" class="p-navigation__sub-link p-dropdown__link">Matrix</a>
          </li>
          
      
          
          <li>
            <a href="https://github.com/canonical/lxd" class="p-navigation__sub-link p-dropdown__link">GitHub</a>
          </li>
          

        </ul>
      </li>

    </ul>
  </div>
</header>
   
    <script>
      document.body.dataset.theme = localStorage.getItem("theme") || "auto";
    </script>
    

<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
  <symbol id="svg-toc" viewBox="0 0 24 24">
    <title>Contents</title>
    <svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 1024 1024">
      <path d="M408 442h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8zm-8 204c0 4.4 3.6 8 8 8h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56zm504-486H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zm0 632H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zM115.4 518.9L271.7 642c5.8 4.6 14.4.5 14.4-6.9V388.9c0-7.4-8.5-11.5-14.4-6.9L115.4 505.1a8.74 8.74 0 0 0 0 13.8z"/>
    </svg>
  </symbol>
  <symbol id="svg-menu" viewBox="0 0 24 24">
    <title>Menu</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-menu">
      <line x1="3" y1="12" x2="21" y2="12"></line>
      <line x1="3" y1="6" x2="21" y2="6"></line>
      <line x1="3" y1="18" x2="21" y2="18"></line>
    </svg>
  </symbol>
  <symbol id="svg-arrow-right" viewBox="0 0 24 24">
    <title>Expand</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-chevron-right">
      <polyline points="9 18 15 12 9 6"></polyline>
    </svg>
  </symbol>
  <symbol id="svg-sun" viewBox="0 0 24 24">
    <title>Light mode</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="feather-sun">
      <circle cx="12" cy="12" r="5"></circle>
      <line x1="12" y1="1" x2="12" y2="3"></line>
      <line x1="12" y1="21" x2="12" y2="23"></line>
      <line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
      <line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
      <line x1="1" y1="12" x2="3" y2="12"></line>
      <line x1="21" y1="12" x2="23" y2="12"></line>
      <line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
      <line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
    </svg>
  </symbol>
  <symbol id="svg-moon" viewBox="0 0 24 24">
    <title>Dark mode</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-moon">
      <path stroke="none" d="M0 0h24v24H0z" fill="none" />
      <path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z" />
    </svg>
  </symbol>
  <symbol id="svg-sun-with-moon" viewBox="0 0 24 24">
    <title>Auto light/dark, in light mode</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round"
      class="icon-custom-derived-from-feather-sun-and-tabler-moon">
      <path style="opacity: 50%" d="M 5.411 14.504 C 5.471 14.504 5.532 14.504 5.591 14.504 C 3.639 16.319 4.383 19.569 6.931 20.352 C 7.693 20.586 8.512 20.551 9.25 20.252 C 8.023 23.207 4.056 23.725 2.11 21.184 C 0.166 18.642 1.702 14.949 4.874 14.536 C 5.051 14.512 5.231 14.5 5.411 14.5 L 5.411 14.504 Z"/>
      <line x1="14.5" y1="3.25" x2="14.5" y2="1.25"/>
      <line x1="14.5" y1="15.85" x2="14.5" y2="17.85"/>
      <line x1="10.044" y1="5.094" x2="8.63" y2="3.68"/>
      <line x1="19" y1="14.05" x2="20.414" y2="15.464"/>
      <line x1="8.2" y1="9.55" x2="6.2" y2="9.55"/>
      <line x1="20.8" y1="9.55" x2="22.8" y2="9.55"/>
      <line x1="10.044" y1="14.006" x2="8.63" y2="15.42"/>
      <line x1="19" y1="5.05" x2="20.414" y2="3.636"/>
      <circle cx="14.5" cy="9.55" r="3.6"/>
    </svg>
  </symbol>
  <symbol id="svg-moon-with-sun" viewBox="0 0 24 24">
    <title>Auto light/dark, in dark mode</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round"
      class="icon-custom-derived-from-feather-sun-and-tabler-moon">
      <path d="M 8.282 7.007 C 8.385 7.007 8.494 7.007 8.595 7.007 C 5.18 10.184 6.481 15.869 10.942 17.24 C 12.275 17.648 13.706 17.589 15 17.066 C 12.851 22.236 5.91 23.143 2.505 18.696 C -0.897 14.249 1.791 7.786 7.342 7.063 C 7.652 7.021 7.965 7 8.282 7 L 8.282 7.007 Z"/>
      <line style="opacity: 50%" x1="18" y1="3.705" x2="18" y2="2.5"/>
      <line style="opacity: 50%" x1="18" y1="11.295" x2="18" y2="12.5"/>
      <line style="opacity: 50%" x1="15.316" y1="4.816" x2="14.464" y2="3.964"/>
      <line style="opacity: 50%" x1="20.711" y1="10.212" x2="21.563" y2="11.063"/>
      <line style="opacity: 50%" x1="14.205" y1="7.5" x2="13.001" y2="7.5"/>
      <line style="opacity: 50%" x1="21.795" y1="7.5" x2="23" y2="7.5"/>
      <line style="opacity: 50%" x1="15.316" y1="10.184" x2="14.464" y2="11.036"/>
      <line style="opacity: 50%" x1="20.711" y1="4.789" x2="21.563" y2="3.937"/>
      <circle style="opacity: 50%" cx="18" cy="7.5" r="2.169"/>
    </svg>
  </symbol>
  <symbol id="svg-pencil" viewBox="0 0 24 24">
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-pencil-code">
      <path d="M4 20h4l10.5 -10.5a2.828 2.828 0 1 0 -4 -4l-10.5 10.5v4" />
      <path d="M13.5 6.5l4 4" />
      <path d="M20 21l2 -2l-2 -2" />
      <path d="M17 17l-2 2l2 2" />
    </svg>
  </symbol>
  <symbol id="svg-eye" viewBox="0 0 24 24">
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-eye-code">
      <path stroke="none" d="M0 0h24v24H0z" fill="none" />
      <path d="M10 12a2 2 0 1 0 4 0a2 2 0 0 0 -4 0" />
      <path
        d="M11.11 17.958c-3.209 -.307 -5.91 -2.293 -8.11 -5.958c2.4 -4 5.4 -6 9 -6c3.6 0 6.6 2 9 6c-.21 .352 -.427 .688 -.647 1.008" />
      <path d="M20 21l2 -2l-2 -2" />
      <path d="M17 17l-2 2l2 2" />
    </svg>
  </symbol>
</svg>

<input type="checkbox" class="sidebar-toggle" name="__navigation" id="__navigation" aria-label="Toggle site navigation sidebar">
<input type="checkbox" class="sidebar-toggle" name="__toc" id="__toc" aria-label="Toggle table of contents sidebar">
<label class="overlay sidebar-overlay" for="__navigation"></label>
<label class="overlay toc-overlay" for="__toc"></label>

<a class="skip-to-content muted-link" href="#furo-main-content">Skip to content</a>



<div class="page">
  <header class="mobile-header">
    <div class="header-left">
      <label class="nav-overlay-icon" for="__navigation">
        <span class="icon"><svg><use href="#svg-menu"></use></svg></span>
      </label>
    </div>
    <div class="header-center">
      <a href="../../"><div class="brand">LXD documentation</div></a>
    </div>
    <div class="header-right">
      <div class="theme-toggle-container theme-toggle-header">
        <button class="theme-toggle" aria-label="Toggle Light / Dark / Auto color theme">
          <svg class="theme-icon-when-auto-light"><use href="#svg-sun-with-moon"></use></svg>
          <svg class="theme-icon-when-auto-dark"><use href="#svg-moon-with-sun"></use></svg>
          <svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
          <svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
        </button>
      </div>
      <label class="toc-overlay-icon toc-header-icon" for="__toc">
        <span class="icon"><svg><use href="#svg-toc"></use></svg></span>
      </label>
    </div>
  </header>
  <aside class="sidebar-drawer">
    <div class="sidebar-container">
      
      <div class="sidebar-sticky"><a class="sidebar-brand" href="../../">
  
  <span class="sidebar-brand-text">LXD documentation</span>
  
</a><form class="sidebar-search-container" method="get" action="../../search/" role="search">
    <input class="sidebar-search" placeholder="Search" name="q" aria-label="Search">
    <input type="submit" value="Go">
    <input type="hidden" name="check_keywords" value="yes">
    <input type="hidden" name="area" value="default">
  </form>
  <div id="searchbox"></div><div class="sidebar-scroll"><div class="sidebar-tree">
  <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../">LXD</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../tutorial/first_steps/">Tutorial</a></li>
<li class="toctree-l1 current has-children"><a class="reference internal" href="../">How-to guides</a><input aria-label="Toggle navigation of How-to guides" checked="" class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" role="switch" type="checkbox"/><label for="toctree-checkbox-1"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul class="current">
<li class="toctree-l2 has-children"><a class="reference internal" href="../../getting_started/">Getting started</a><input aria-label="Toggle navigation of Getting started" class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" role="switch" type="checkbox"/><label for="toctree-checkbox-2"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../../installing/">Install LXD</a></li>
<li class="toctree-l3"><a class="reference internal" href="../initialize/">Initialize LXD</a></li>
<li class="toctree-l3"><a class="reference internal" href="../access_ui/">Access the UI</a></li>
<li class="toctree-l3"><a class="reference internal" href="../access_documentation/">Access documentation locally</a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../operation/">LXD server and client</a><input aria-label="Toggle navigation of LXD server and client" class="toctree-checkbox" id="toctree-checkbox-3" name="toctree-checkbox-3" role="switch" type="checkbox"/><label for="toctree-checkbox-3"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../server_expose/">Expose LXD to the network</a></li>
<li class="toctree-l3"><a class="reference internal" href="../server_configure/">Configure the LXD server</a></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../oidc/">Configure single sign-on with OIDC</a><input aria-label="Toggle navigation of Configure single sign-on with OIDC" class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" role="switch" type="checkbox"/><label for="toctree-checkbox-4"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l4"><a class="reference internal" href="../oidc_auth0/">How to configure Auth0</a></li>
<li class="toctree-l4"><a class="reference internal" href="../oidc_ory/">How to configure Ory Hydra</a></li>
<li class="toctree-l4"><a class="reference internal" href="../oidc_keycloak/">How to configure Keycloak</a></li>
<li class="toctree-l4"><a class="reference internal" href="../oidc_entra_id/">How to configure Entra ID</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../remotes/">Add remote servers</a></li>
<li class="toctree-l3"><a class="reference internal" href="../lxc_alias/">Add command aliases</a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../instances/">Instances</a><input aria-label="Toggle navigation of Instances" class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" role="switch" type="checkbox"/><label for="toctree-checkbox-5"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../instances_create/">Create instances</a></li>
<li class="toctree-l3"><a class="reference internal" href="../instances_configure/">Configure instances</a></li>
<li class="toctree-l3"><a class="reference internal" href="../instances_manage/">Manage instances</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../profiles/">Use profiles</a></li>
<li class="toctree-l3"><a class="reference internal" href="../instances_troubleshoot/">Troubleshoot errors</a></li>
<li class="toctree-l3"><a class="reference internal" href="../instances_ubuntu_pro_attach/">Auto attach Ubuntu Pro</a></li>
<li class="toctree-l3"><a class="reference internal" href="../instances_access_files/">Access files</a></li>
<li class="toctree-l3"><a class="reference internal" href="../instances_console/">Access the console</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../instance-exec/">Run commands</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../cloud-init/">Use cloud-init</a></li>
<li class="toctree-l3"><a class="reference internal" href="../instances_routed_nic_vm/">Add a routed NIC to a VM</a></li>
<li class="toctree-l3"><a class="reference internal" href="../instances_backup/">Back up instances</a></li>
<li class="toctree-l3"><a class="reference internal" href="../instances_migrate/">Migrate instances</a></li>
<li class="toctree-l3"><a class="reference internal" href="../import_machines_to_instances/">Import existing machines</a></li>
<li class="toctree-l3"><a class="reference internal" href="../container_gpu_passthrough_with_docker/">Pass NVIDIA GPUs</a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../images/">Images</a><input aria-label="Toggle navigation of Images" class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" role="switch" type="checkbox"/><label for="toctree-checkbox-6"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../images_remote/">Use remote images</a></li>
<li class="toctree-l3"><a class="reference internal" href="../images_manage/">Manage images</a></li>
<li class="toctree-l3"><a class="reference internal" href="../images_profiles/">Associate profiles</a></li>
<li class="toctree-l3"><a class="reference internal" href="../images_copy/">Copy and import images</a></li>
<li class="toctree-l3"><a class="reference internal" href="../images_create/">Create images</a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../projects/">Projects</a><input aria-label="Toggle navigation of Projects" class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" role="switch" type="checkbox"/><label for="toctree-checkbox-7"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../projects_create/">Create and configure</a></li>
<li class="toctree-l3"><a class="reference internal" href="../projects_work/">Work with projects</a></li>
<li class="toctree-l3"><a class="reference internal" href="../projects_confine/">Confine users to projects</a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../storage/">Storage</a><input aria-label="Toggle navigation of Storage" class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" role="switch" type="checkbox"/><label for="toctree-checkbox-8"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../storage_pools/">Manage pools</a></li>
<li class="toctree-l3"><a class="reference internal" href="../storage_volumes/">Manage volumes</a></li>
<li class="toctree-l3"><a class="reference internal" href="../storage_buckets/">Manage buckets</a></li>
<li class="toctree-l3"><a class="reference internal" href="../storage_create_instance/">Create an instance in a pool</a></li>
<li class="toctree-l3"><a class="reference internal" href="../storage_backup_volume/">Back up a volume</a></li>
<li class="toctree-l3"><a class="reference internal" href="../storage_move_volume/">Move or copy a volume</a></li>
<li class="toctree-l3"><a class="reference internal" href="../storage_csi/">Use the LXD CSI driver with Kubernetes</a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../networks/">Networking</a><input aria-label="Toggle navigation of Networking" class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" role="switch" type="checkbox"/><label for="toctree-checkbox-9"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../network_create/">Create a network</a></li>
<li class="toctree-l3"><a class="reference internal" href="../network_configure/">Configure a network</a></li>
<li class="toctree-l3"><a class="reference internal" href="../network_bgp/">Configure as BGP server</a></li>
<li class="toctree-l3"><a class="reference internal" href="../network_acls/">Configure network ACLs</a></li>
<li class="toctree-l3"><a class="reference internal" href="../network_forwards/">Configure forwards</a></li>
<li class="toctree-l3"><a class="reference internal" href="../network_zones/">Configure network zones</a></li>
<li class="toctree-l3"><a class="reference internal" href="../network_bridge_firewalld/">Configure your firewall</a></li>
<li class="toctree-l3"><a class="reference internal" href="../network_bridge_resolved/">Integrate with resolved</a></li>
<li class="toctree-l3"><a class="reference internal" href="../network_ovn_setup/">Set up OVN</a></li>
<li class="toctree-l3"><a class="reference internal" href="../network_load_balancers/">Configure load balancers</a></li>
<li class="toctree-l3"><a class="reference internal" href="../network_ovn_peers/">Configure peer routing</a></li>
<li class="toctree-l3"><a class="reference internal" href="../network_ipam/">Display IPAM information</a></li>
</ul>
</li>
<li class="toctree-l2 current has-children"><a class="reference internal" href="../../clustering/">Clustering</a><input aria-label="Toggle navigation of Clustering" checked="" class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" role="switch" type="checkbox"/><label for="toctree-checkbox-10"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="../cluster_form/">Form a cluster</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cluster_manage/">Manage a cluster</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cluster_config_networks/">Configure networks</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cluster_config_storage/">Configure storage</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cluster_manage_instance/">Manage instances</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cluster_groups/">Set up cluster groups</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cluster_placement_groups/">Use placement groups</a></li>
<li class="toctree-l3 current current-page"><a class="current reference internal" href="#">Recover a cluster</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cluster_vip/">Set up a highly available virtual IP</a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../production-setup/">Production setup</a><input aria-label="Toggle navigation of Production setup" class="toctree-checkbox" id="toctree-checkbox-11" name="toctree-checkbox-11" role="switch" type="checkbox"/><label for="toctree-checkbox-11"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../benchmark_performance/">Benchmark performance</a></li>
<li class="toctree-l3"><a class="reference internal" href="../network_increase_bandwidth/">Increase bandwidth</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../metrics/">Monitor metrics</a></li>
<li class="toctree-l3"><a class="reference internal" href="../logs_loki/">Send logs to Loki</a></li>
<li class="toctree-l3"><a class="reference internal" href="../grafana/">Set up Grafana</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../backup/">Back up a server</a></li>
<li class="toctree-l3"><a class="reference internal" href="../disaster_recovery/">Recover instances</a></li>
<li class="toctree-l3"><a class="reference internal" href="../disaster_recovery_replication/">Disaster recovery with storage replication</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../snap/">Manage the snap</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security_harden/">Harden security</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../troubleshoot/">Troubleshooting</a><input aria-label="Toggle navigation of Troubleshooting" class="toctree-checkbox" id="toctree-checkbox-12" name="toctree-checkbox-12" role="switch" type="checkbox"/><label for="toctree-checkbox-12"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../network_bridge_firewalld/">Configure your firewall</a></li>
<li class="toctree-l3"><a class="reference internal" href="../instances_troubleshoot/">Troubleshoot instances</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dqlite_troubleshoot/">Troubleshoot Dqlite</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../debugging/">Debug LXD</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../faq/">Frequently asked</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../support/">Get support</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../contributing/">Contribute to LXD</a></li>
<li class="toctree-l2"><a class="reference internal" href="../auth_bearer/">How to authenticate to the LXD API using bearer tokens</a></li>
<li class="toctree-l2"><a class="reference internal" href="../devlxd_authenticate/">How to authenticate to the DevLXD API</a></li>
</ul>
</li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../../explanation/">Explanation</a><input aria-label="Toggle navigation of Explanation" class="toctree-checkbox" id="toctree-checkbox-13" name="toctree-checkbox-13" role="switch" type="checkbox"/><label for="toctree-checkbox-13"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l2"><a class="reference internal" href="../../explanation/lxd_lxc/"><code class="docutils literal notranslate"><span class="pre">lxd</span></code> and <code class="docutils literal notranslate"><span class="pre">lxc</span></code></a></li>
<li class="toctree-l2"><a class="reference internal" href="../../explanation/instances/">Containers and VMs</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../image-handling/">Local and remote images</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../explanation/storage/">Storage pools, volumes, and buckets</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../explanation/networks/">Networking setups</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../database/">The LXD Dqlite database</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../explanation/lxc_show_info/"><code class="docutils literal notranslate"><span class="pre">lxc</span></code> <code class="docutils literal notranslate"><span class="pre">show</span></code> and <code class="docutils literal notranslate"><span class="pre">info</span></code></a></li>
<li class="toctree-l2"><a class="reference internal" href="../../authentication/">Remote API authentication</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../explanation/authorization/">Remote API authorization</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../explanation/projects/">Instances grouping with projects</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../explanation/clusters/">Clusters</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../explanation/performance_tuning/">Performance tuning</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../explanation/security/">Security</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../explanation/bpf/">Privilege delegation using BPF Token</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../explanation/csi/">The LXD CSI driver</a></li>
</ul>
</li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../../reference/">Reference</a><input aria-label="Toggle navigation of Reference" class="toctree-checkbox" id="toctree-checkbox-14" name="toctree-checkbox-14" role="switch" type="checkbox"/><label for="toctree-checkbox-14"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l2"><a class="reference internal" href="../../requirements/">Requirements</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../architectures/">Architectures</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../reference/release-notes/">Release notes</a><input aria-label="Toggle navigation of Release notes" class="toctree-checkbox" id="toctree-checkbox-15" name="toctree-checkbox-15" role="switch" type="checkbox"/><label for="toctree-checkbox-15"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../../reference/release-notes/release-notes-6.7/">LXD 6.7</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/release-notes/release-notes-6.6/">LXD 6.6</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/releases-snap/">Releases and snap</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/remote_image_servers/">Remote image servers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/image_format/">Image format</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../guest-os-compatibility/">Guest OS compatibility</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../container-environment/">Container environment</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../config-options/">Configuration option index</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../server/">Server configuration</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../explanation/instance_config/">Instance configuration</a><input aria-label="Toggle navigation of Instance configuration" class="toctree-checkbox" id="toctree-checkbox-16" name="toctree-checkbox-16" role="switch" type="checkbox"/><label for="toctree-checkbox-16"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../../reference/instance_properties/">Instance properties</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/instance_options/">Instance options</a></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../reference/devices/">Devices</a><input aria-label="Toggle navigation of Devices" class="toctree-checkbox" id="toctree-checkbox-17" name="toctree-checkbox-17" role="switch" type="checkbox"/><label for="toctree-checkbox-17"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l4"><a class="reference internal" href="../../reference/standard_devices/">Standard devices</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/devices_none/">Type: <code class="docutils literal notranslate"><span class="pre">none</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/devices_nic/">Type: <code class="docutils literal notranslate"><span class="pre">nic</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/devices_disk/">Type: <code class="docutils literal notranslate"><span class="pre">disk</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/devices_unix_char/">Type: <code class="docutils literal notranslate"><span class="pre">unix-char</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/devices_unix_block/">Type: <code class="docutils literal notranslate"><span class="pre">unix-block</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/devices_usb/">Type: <code class="docutils literal notranslate"><span class="pre">usb</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/devices_gpu/">Type: <code class="docutils literal notranslate"><span class="pre">gpu</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/devices_infiniband/">Type: <code class="docutils literal notranslate"><span class="pre">infiniband</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/devices_proxy/">Type: <code class="docutils literal notranslate"><span class="pre">proxy</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/devices_unix_hotplug/">Type: <code class="docutils literal notranslate"><span class="pre">unix-hotplug</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/devices_tpm/">Type: <code class="docutils literal notranslate"><span class="pre">tpm</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/devices_pci/">Type: <code class="docutils literal notranslate"><span class="pre">pci</span></code></a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/instance_units/">Units for storage and network limits</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/preseed_yaml_fields/">Preseed YAML file fields</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/projects/">Project configuration</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../reference/storage_drivers/">Storage drivers</a><input aria-label="Toggle navigation of Storage drivers" class="toctree-checkbox" id="toctree-checkbox-18" name="toctree-checkbox-18" role="switch" type="checkbox"/><label for="toctree-checkbox-18"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../../reference/storage_dir/">Directory - <code class="docutils literal notranslate"><span class="pre">dir</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/storage_btrfs/">Btrfs - <code class="docutils literal notranslate"><span class="pre">btrfs</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/storage_lvm/">LVM - <code class="docutils literal notranslate"><span class="pre">lvm</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/storage_zfs/">ZFS - <code class="docutils literal notranslate"><span class="pre">zfs</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/storage_ceph/">Ceph RBD - <code class="docutils literal notranslate"><span class="pre">ceph</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/storage_powerflex/">Dell PowerFlex - <code class="docutils literal notranslate"><span class="pre">powerflex</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/storage_pure/">Pure Storage - <code class="docutils literal notranslate"><span class="pre">pure</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/storage_alletra/">HPE Alletra - <code class="docutils literal notranslate"><span class="pre">alletra</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/storage_cephfs/">CephFS - <code class="docutils literal notranslate"><span class="pre">cephfs</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/storage_cephobject/">Ceph Object - <code class="docutils literal notranslate"><span class="pre">cephobject</span></code></a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../reference/networks/">Networks</a><input aria-label="Toggle navigation of Networks" class="toctree-checkbox" id="toctree-checkbox-19" name="toctree-checkbox-19" role="switch" type="checkbox"/><label for="toctree-checkbox-19"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../../reference/network_bridge/">Bridge network</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/network_ovn/">OVN network</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/network_macvlan/">Macvlan network</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/network_physical/">Physical network</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/network_sriov/">SR-IOV network</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/cluster_member_config/">Cluster configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/placement_groups/">Placement group configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/server_settings/">Production server settings</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/provided_metrics/">Provided metrics</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/permissions/">Permissions</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../restapi_landing/">REST API</a><input aria-label="Toggle navigation of REST API" class="toctree-checkbox" id="toctree-checkbox-20" name="toctree-checkbox-20" role="switch" type="checkbox"/><label for="toctree-checkbox-20"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../../rest-api/">Main API documentation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../api/">Main API specification</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../api-extensions/">Main API extensions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../events/">Events API documentation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../dev-lxd/">Instance API</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/driver_csi/">LXD CSI driver reference</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../reference/manpages/">Man pages</a><input aria-label="Toggle navigation of Man pages" class="toctree-checkbox" id="toctree-checkbox-21" name="toctree-checkbox-21" role="switch" type="checkbox"/><label for="toctree-checkbox-21"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../../reference/manpages/lxc/"><code class="docutils literal notranslate"><span class="pre">lxc</span></code></a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../internals/">Internals</a><input aria-label="Toggle navigation of Internals" class="toctree-checkbox" id="toctree-checkbox-22" name="toctree-checkbox-22" role="switch" type="checkbox"/><label for="toctree-checkbox-22"><span class="icon"><svg><use href="#svg-arrow-right"></use></svg></span></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../../environment/">Environment variables</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/uefi_variables/">UEFI variables for VMs</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../daemon-behavior/">Daemon behavior</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../syscall-interception/">System call interception</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../userns-idmap/">User namespace setup</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/ovn-internals/">OVN implementation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/vm_live_migration_internals/">VM live migration implementation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/dqlite-internals/">Dqlite</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference external" href="https://github.com/canonical/lxd">Project repository</a></li>
<li class="toctree-l2"><a class="reference external" href="https://images.lxd.canonical.com">Image server</a></li>
</ul>
</li>
</ul>

</div>
</div>

      </div>
      
    </div>
  </aside>
  <div class="main">
    <div class="content">
      <div class="article-container">
        <a href="#" class="back-to-top muted-link">
          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24">
            <path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12z"></path>
          </svg>
          <span>Back to top</span>
        </a>
        <div class="content-icon-container">
          <div class="edit-this-page">
  <a class="muted-link" href="https://github.com/canonical/lxd/edit/main/doc/howto/cluster_recover.md" title="Contribute to this page">
    <svg><use href="#svg-pencil"></use></svg>
    <span class="visually-hidden">Contribute to this page</span>
  </a>
</div><div class="theme-toggle-container theme-toggle-content">
            <button class="theme-toggle" aria-label="Toggle Light / Dark / Auto color theme">
              <svg class="theme-icon-when-auto-light"><use href="#svg-sun-with-moon"></use></svg>
              <svg class="theme-icon-when-auto-dark"><use href="#svg-moon-with-sun"></use></svg>
              <svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
              <svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
            </button>
          </div>
          <label class="toc-overlay-icon toc-content-icon" for="__toc">
            <span class="icon"><svg><use href="#svg-toc"></use></svg></span>
          </label>
        </div>
        <article role="main" id="furo-main-content">
          <section id="how-to-recover-a-cluster">
<span id="cluster-recover"></span><h1>How to recover a cluster<a class="headerlink" href="#how-to-recover-a-cluster" title="Link to this heading">¶</a></h1>
<p>It might happen that one or several members of your cluster go offline or become unreachable.
If too many cluster members go offline, no operations will be possible on the cluster.
See <a class="reference internal" href="../../explanation/clusters/#clustering-offline-members"><span class="std std-ref">Offline members and fault tolerance</span></a> and <a class="reference internal" href="../cluster_manage/#cluster-automatic-evacuation"><span class="std std-ref">Cluster healing</span></a> for more information.</p>
<p>If you can bring the offline cluster members back up, operation resumes as normal.
If the cluster members are lost permanently (e.g. disk failure), it is possible
to recover any remaining cluster members.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>When your cluster is in a state that needs recovery, most <code class="docutils literal notranslate"><span class="pre">lxc</span></code> commands do not
work because the LXD database does not respond when a majority of database
voters are inaccessible.</p>
<p>The commands to recover a cluster are provided directly by the LXD daemon (<code class="docutils literal notranslate"><span class="pre">lxd</span></code>)
because they modify database files directly instead of making requests to the
LXD daemon.</p>
<p>Run <code class="docutils literal notranslate"><span class="pre">lxd</span> <span class="pre">cluster</span> <span class="pre">--help</span></code> for an overview of all available commands.</p>
</div>
<section id="database-members">
<h2>Database members<a class="headerlink" href="#database-members" title="Link to this heading">¶</a></h2>
<p>Every LXD cluster has a specific number of members (configured through <a class="configref reference internal" href="../../server/#server-cluster:cluster.max_voters"><code class="docutils literal notranslate"><span class="pre">cluster.max_voters</span></code></a>) that serve as voting members of the distributed database.
If you lose a majority of these cluster members (for example, you have a three-member cluster and you lose two members), the cluster loses quorum and becomes unavailable.</p>
<p>To determine which members have (or had) database roles, log on to any surviving member of your cluster and run the following command:</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>sudo lxd cluster list-database
</pre></div>
</div>
</section>
<section id="recover-from-quorum-loss">
<h2>Recover from quorum loss<a class="headerlink" href="#recover-from-quorum-loss" title="Link to this heading">¶</a></h2>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>LXD automatically takes a backup of the database before making changes (see <a class="reference internal" href="#automated-backups"><span class="std std-ref">Automated Backups</span></a>).</p>
</div>
<p>If only one cluster member with the database role survives, complete the following
steps. See <a class="reference internal" href="#reconfigure-the-cluster">Reconfigure the cluster</a> below for recovering
more than one member.</p>
<ol class="arabic">
<li><p>Make sure that the LXD daemon is not running on the machine.
For example, if you’re using the snap:</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>sudo snap stop lxd
</pre></div>
</div>
</li>
<li><p>Use the following command to reconfigure the database:</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>sudo lxd cluster recover-from-quorum-loss
</pre></div>
</div>
</li>
<li><p>Start the LXD daemon again. For example, if you’re using the snap:</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>sudo snap start lxd
</pre></div>
</div>
</li>
</ol>
<p>The database should now be back online.
No information has been deleted from the database.
All information about the cluster members that you have lost is still there, including the metadata about their instances.
This can help you with further recovery steps if you need to re-create the lost instances.</p>
<p>To permanently delete the cluster members that you have lost, force-remove them.
See <a class="reference internal" href="../cluster_manage/#cluster-manage-delete-members"><span class="std std-ref">Delete cluster members</span></a>.</p>
</section>
<section id="reconfigure-the-cluster">
<h2>Reconfigure the cluster<a class="headerlink" href="#reconfigure-the-cluster" title="Link to this heading">¶</a></h2>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>LXD automatically takes a backup of the database before making changes (see <a class="reference internal" href="#automated-backups"><span class="std std-ref">Automated Backups</span></a>).</p>
</div>
<p>If some members of your cluster are no longer reachable, or if the cluster itself is unreachable due to a change in IP address or listening port number, you can reconfigure the cluster.</p>
<p>To do so, choose the <a class="reference internal" href="#up-to-date-cluster-member"><span class="std std-ref">most up-to-date database member</span></a> to edit the cluster configuration.
Once the cluster edit is complete you will need to manually copy the reconfigured global database to every other surviving member.</p>
<p>You can change the IP addresses or listening port numbers for each member as required.
You cannot add or remove any members during this process.
The cluster configuration must contain the description of the full cluster.</p>
<p>You can edit the <a class="reference internal" href="../../explanation/clusters/#clustering-member-roles"><span class="std std-ref">Member roles</span></a> of the members, but with the following limitations:</p>
<ul class="simple">
<li><p>A cluster member that does not have a <code class="docutils literal notranslate"><span class="pre">database*</span></code> role cannot become a voter, because it might lack a global database.</p></li>
<li><p>At least two members must remain voters (except in the case of a two-member cluster, where one voter suffices), or there will be no quorum.</p></li>
</ul>
<p>Before performing the recovery, stop the LXD daemon on all surviving cluster members.
For example, if you’re using the snap:</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>sudo snap stop lxd
</pre></div>
</div>
<p>Complete the following steps on one database member:</p>
<ol class="arabic">
<li><p>Run the following command:</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>sudo lxd cluster edit
</pre></div>
</div>
</li>
<li><p>Edit the YAML representation of the information that this cluster member has about the rest of the cluster:</p>
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="c1"># Latest dqlite segment ID: 1234</span>

<span class="nt">members</span><span class="p">:</span>
<span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">1</span><span class="w">             </span><span class="c1"># Internal ID of the member (Read-only)</span>
<span class="w">    </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">server1</span><span class="w">     </span><span class="c1"># Name of the cluster member (Read-only)</span>
<span class="w">    </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">192.0.2.10:8443</span><span class="w"> </span><span class="c1"># Last known address of the member (Writeable)</span>
<span class="w">    </span><span class="nt">role</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">voter</span><span class="w">              </span><span class="c1"># Last known role of the member (Writeable)</span>
<span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">2</span><span class="w">             </span><span class="c1"># Internal ID of the member (Read-only)</span>
<span class="w">    </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">server2</span><span class="w">     </span><span class="c1"># Name of the cluster member (Read-only)</span>
<span class="w">    </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">192.0.2.11:8443</span><span class="w"> </span><span class="c1"># Last known address of the member (Writeable)</span>
<span class="w">    </span><span class="nt">role</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">stand-by</span><span class="w">           </span><span class="c1"># Last known role of the member (Writeable)</span>
<span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">3</span><span class="w">             </span><span class="c1"># Internal ID of the member (Read-only)</span>
<span class="w">    </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">server3</span><span class="w">     </span><span class="c1"># Name of the cluster member (Read-only)</span>
<span class="w">    </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">192.0.2.12:8443</span><span class="w"> </span><span class="c1"># Last known address of the member (Writeable)</span>
<span class="w">    </span><span class="nt">role</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">spare</span><span class="w">              </span><span class="c1"># Last known role of the member (Writeable)</span>
</pre></div>
</div>
<p>You can edit the addresses and the roles.</p>
</li>
<li><p>When the cluster configuration has been changed on one member, LXD will create
a tarball of the global database (<code class="docutils literal notranslate"><span class="pre">/var/snap/lxd/common/lxd/database/lxd_recovery_db.tar.gz</span></code>
for snap installations or <code class="docutils literal notranslate"><span class="pre">/var/lib/lxd/database/lxd_recovery_db.tar.gz</span></code>).
Copy this recovery tarball to the same path on all remaining cluster members.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>The tarball can be removed from the first member after it is generated, but
it does not have to be.</p>
</div>
</li>
<li><p>Once the tarball has been copied to all remaining cluster members, start the
LXD daemon on all members again. LXD will load the recovery tarball on startup.</p>
<p>If you’re using the snap:</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>sudo snap start lxd
</pre></div>
</div>
</li>
</ol>
<p>The cluster should now be fully available again with all surviving members reporting in.
No information has been deleted from the database.
All information about the cluster members and their instances is still there.</p>
</section>
<section id="automated-backups">
<span id="id1"></span><h2>Automated Backups<a class="headerlink" href="#automated-backups" title="Link to this heading">¶</a></h2>
<p>LXD automatically creates a backup of the database before making changes during
recovery. The backup is just a tarball of <code class="docutils literal notranslate"><span class="pre">/var/snap/lxd/common/lxd/database</span></code>
(for snap users) or <code class="docutils literal notranslate"><span class="pre">/var/lib/lxd/lxd/database</span></code> (otherwise). To reset the state
of the database in case of a failure, simply delete the database directory and
unpack the tarball in its place:</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>cd /var/snap/lxd/common/lxd
sudo rm -r database
sudo tar -xf db_backup.TIMESTAMP.tar.gz
</pre></div>
</div>
</section>
<section id="find-the-most-up-to-date-cluster-member">
<span id="up-to-date-cluster-member"></span><h2>Find the most up-to-date cluster member<a class="headerlink" href="#find-the-most-up-to-date-cluster-member" title="Link to this heading">¶</a></h2>
<p>On every shutdown, LXD’s <a class="reference internal" href="../../explanation/clusters/#clustering-member-roles"><span class="std std-ref">database members</span></a> log
the Raft term and index:</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>Dqlite last entry    index=1039 term=672
</pre></div>
</div>
<p>To determine which database member is most up to date:</p>
<ul class="simple">
<li><p>If two members have different terms, the member with the higher term is more up to date.</p></li>
<li><p>If two members have the same term, the member with the higher index is more up to date.</p></li>
</ul>
</section>
<section id="manually-alter-raft-membership">
<h2>Manually alter Raft membership<a class="headerlink" href="#manually-alter-raft-membership" title="Link to this heading">¶</a></h2>
<p>In some situations, you might need to manually alter the Raft membership configuration of the cluster because of some unexpected behavior.</p>
<p>For example, if you have a cluster member that was removed uncleanly, it might not show up in <a class="reference internal" href="../../reference/manpages/lxc/cluster/list/#lxc-cluster-list-md"><span class="std std-ref"><code class="docutils literal notranslate"><span class="pre">lxc</span> <span class="pre">cluster</span> <span class="pre">list</span></code></span></a> but still be part of the Raft configuration.
To see the Raft configuration, run the following command:</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>lxd sql local &quot;SELECT * FROM raft_nodes&quot;
</pre></div>
</div>
<p>In that case, run the following command to remove the leftover node:</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>lxd cluster remove-raft-node &lt;address&gt;
</pre></div>
</div>
</section>
</section>

        </article>
      </div>
      <footer>
        
   

<div class="related-pages">
  
  
      
  
  
  <a class="next-page" href="../cluster_vip/">
        <div class="page-info">
          <div class="context">
            <span>Next</span>
          </div>
          <div class="title">How to set up a highly available virtual IP for clusters</div>
        </div>
        <svg class="furo-related-icon"><use href="#svg-arrow-right"></use></svg>
      </a>
    <a class="prev-page" href="../cluster_placement_groups/">
        <svg class="furo-related-icon"><use href="#svg-arrow-right"></use></svg>
        <div class="page-info">
          <div class="context">
            <span>Previous</span>
          </div>
          
            <div class="title">How to use placement groups</div>
          
        </div>
      </a>
</div>
<div class="bottom-of-page">
  <div class="left-details">
    <div class="copyright">
        &copy; 2014-2026 AGPL-3.0, LXD contributors 
    </div><div class="last-updated">
      Last updated on Dec 09, 2024</div>
  </div>
  <div class="right-details">
    
      <a href="" class="js-revoke-cookie-manager muted-link">Manage your tracker settings</a>
</div>

      </footer>
    </div>
    <aside class="toc-drawer">
      
<div class="toc-sticky toc-scroll">
   
    <div class="toc-title-container">
      <span class="toc-title">
       Contents
      </span>
    </div>
    <div class="toc-tree-container">
      <div class="toc-tree">
        <ul>
<li><a class="reference internal" href="#">How to recover a cluster</a><ul>
<li><a class="reference internal" href="#database-members">Database members</a></li>
<li><a class="reference internal" href="#recover-from-quorum-loss">Recover from quorum loss</a></li>
<li><a class="reference internal" href="#reconfigure-the-cluster">Reconfigure the cluster</a></li>
<li><a class="reference internal" href="#automated-backups">Automated Backups</a></li>
<li><a class="reference internal" href="#find-the-most-up-to-date-cluster-member">Find the most up-to-date cluster member</a></li>
<li><a class="reference internal" href="#manually-alter-raft-membership">Manually alter Raft membership</a></li>
</ul>
</li>
</ul>

      </div>
    </div>
   
    
  </div>

    </aside>
  </div>
</div><script src="../../_static/jquery.js?v=5d32c60e"></script>
    <script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
    <script src="../../_static/documentation_options.js?v=a5603611"></script>
    <script src="../../_static/doctools.js?v=9a2dae69"></script>
    <script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
    <script src="../../_static/scripts/furo.js?v=46bd48cc"></script>
    <script src="../../_static/clipboard.min.js?v=a7894cd8"></script>
    <script src="../../_static/copybutton.js?v=b01cb6f2"></script>
    <script src="../../_static/config-options.js"></script>
    <script src="../../_static/design-tabs.js?v=f930bc37"></script>
    <script src="../../_static/js/bundle.js?v=a4d88309"></script>
    <script src="../../_static/header-nav.js?v=e117ad08"></script>
    <script src="../../_static/github_issue_links.js?v=32bb732f"></script>
    
<script>
  const github_url = "https://github.com/canonical/lxd";
</script>
</body>
</html>