Uploaded image for project: 'Percona Monitoring and Management'
  1. Percona Monitoring and Management
  2. PMM-5014

LVM Volumes wrongly sized on new AWS deployment

Details

    • Bug
    • Status: Done
    • High
    • Resolution: Fixed
    • 1.17.2
    • 2.2.0, 1.17.3
    • AMI, Packaging
    • None
    • 2
    • Platform Sprint 4, Platform Sprint 5
    • Yes
    • Yes
    • Yes

    Description

      I deployed a new instance from AWS AMI version 1.17.2.

      It stopped working in a few hours. It appears it ran out of disk space. 

      Closer examination revealed that there is an incorrect LVM sizing. Also attached is the latest messages syslog.

      Details below:

       

      [[email protected] srv]# lsblk
      NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
      nvme0n1 259:1 0 8G 0 disk
      └─nvme0n1p1 259:2 0 8G 0 part /
      nvme1n1 259:0 0 100G 0 disk
      ├─DataVG-ThinPool_tmeta 253:0 0 100M 0 lvm
      │ └─DataVG-ThinPool-tpool 253:2 0 99.8G 0 lvm
      │ ├─DataVG-ThinPool 253:3 0 99.8G 0 lvm
      │ └─DataVG-DataLV 253:4 0 1G 0 lvm /srv
      └─DataVG-ThinPool_tdata 253:1 0 99.8G 0 lvm
      └─DataVG-ThinPool-tpool 253:2 0 99.8G 0 lvm
      ├─DataVG-ThinPool 253:3 0 99.8G 0 lvm
      └─DataVG-DataLV 253:4 0 1G 0 lvm /srv

       

      [[email protected] srv]# mount
      sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime,seclabel)
      proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
      devtmpfs on /dev type devtmpfs (rw,nosuid,seclabel,size=896164k,nr_inodes=224041,mode=755)
      securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime)
      tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev,seclabel)
      devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,seclabel,gid=5,mode=620,ptmxmode=000)
      tmpfs on /run type tmpfs (rw,nosuid,nodev,seclabel,mode=755)
      tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,seclabel,mode=755)
      cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd)
      pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime)
      cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,net_prio,net_cls)
      cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,cpuacct,cpu)
      cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,freezer)
      cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,pids)
      cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,cpuset)
      cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,memory)
      cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,perf_event)
      cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,hugetlb)
      cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,blkio)
      cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,devices)
      configfs on /sys/kernel/config type configfs (rw,relatime)
      /dev/nvme0n1p1 on / type xfs (rw,relatime,seclabel,attr2,inode64,noquota)
      rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime)
      selinuxfs on /sys/fs/selinux type selinuxfs (rw,relatime)
      hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime,seclabel)
      debugfs on /sys/kernel/debug type debugfs (rw,relatime)
      systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11025)
      mqueue on /dev/mqueue type mqueue (rw,relatime,seclabel)
      /dev/mapper/DataVG-DataLV on /srv type xfs (rw,relatime,seclabel,attr2,inode64,logbsize=64k,sunit=128,swidth=128,noquota)
      tmpfs on /run/user/1001 type tmpfs (rw,nosuid,nodev,relatime,seclabel,size=184316k,mode=700,uid=1001,gid=1001)

       

      [[email protected] srv]# df -lh
      Filesystem Size Used Avail Use% Mounted on
      devtmpfs 876M 0 876M 0% /dev
      tmpfs 900M 0 900M 0% /dev/shm
      tmpfs 900M 118M 783M 14% /run
      tmpfs 900M 0 900M 0% /sys/fs/cgroup
      /dev/nvme0n1p1 8.0G 3.7G 4.4G 46% /
      /dev/mapper/DataVG-DataLV 1014M 1014M 20K 100% /srv
      tmpfs 180M 0 180M 0% /run/user/1001

       

      [[email protected] ~]# lvdisplay
      — Logical volume —
      LV Name ThinPool
      VG Name DataVG
      LV UUID 725l1r-f3go-vL86-eQNd-EHIS-44DT-NlwHq2
      LV Write Access read/write
      LV Creation host, time ip-172-30-5-209.ec2.internal, 2019-10-24 13:42:01 +0000
      LV Pool metadata ThinPool_tmeta
      LV Pool data ThinPool_tdata
      LV Status available

      1. open 2
        LV Size 99.80 GiB
        Allocated pool data 0.97%
        Allocated metadata 10.91%
        Current LE 25549
        Segments 1
        Allocation inherit
        Read ahead sectors auto
      • currently set to 8192
        Block device 253:2

      — Logical volume —
      LV Path /dev/DataVG/DataLV
      LV Name DataLV
      VG Name DataVG
      LV UUID 5ctyw7-01bK-Obu8-iQMc-a0fM-gIYo-9iF6MW
      LV Write Access read/write
      LV Creation host, time ip-172-30-5-209.ec2.internal, 2019-10-24 13:42:03 +0000
      LV Pool name ThinPool
      LV Status available

      1. open 1
        LV Size 1.00 GiB
        Mapped size 97.11%
        Current LE 256
        Segments 1
        Allocation inherit
        Read ahead sectors auto
      • currently set to 8192
        Block device 253:4

       

      [[email protected] ~]# vgdisplay
      — Volume group —
      VG Name DataVG
      System ID
      Format lvm2
      Metadata Areas 1
      Metadata Sequence No 657
      VG Access read/write
      VG Status resizable
      MAX LV 0
      Cur LV 2
      Open LV 1
      Max PV 0
      Cur PV 1
      Act PV 1
      VG Size <100.00 GiB
      PE Size 4.00 MiB
      Total PE 25599
      Alloc PE / Size 25599 / <100.00 GiB
      Free PE / Size 0 / 0
      VG UUID s9hIMr-q4km-K14c-Z2oD-41nE-2NRF-Rc8tuj

       

      [[email protected] ~]# lvdisplay
      — Logical volume —
      LV Name ThinPool
      VG Name DataVG
      LV UUID 725l1r-f3go-vL86-eQNd-EHIS-44DT-NlwHq2
      LV Write Access read/write
      LV Creation host, time ip-172-30-5-209.ec2.internal, 2019-10-24 13:42:01 +0000
      LV Pool metadata ThinPool_tmeta
      LV Pool data ThinPool_tdata
      LV Status available

      1. open 2
        LV Size 99.80 GiB
        Allocated pool data 0.97%
        Allocated metadata 10.91%
        Current LE 25549
        Segments 1
        Allocation inherit
        Read ahead sectors auto
      • currently set to 8192
        Block device 253:2

      — Logical volume —
      LV Path /dev/DataVG/DataLV
      LV Name DataLV
      VG Name DataVG
      LV UUID 5ctyw7-01bK-Obu8-iQMc-a0fM-gIYo-9iF6MW
      LV Write Access read/write
      LV Creation host, time ip-172-30-5-209.ec2.internal, 2019-10-24 13:42:03 +0000
      LV Pool name ThinPool
      LV Status available

      1. open 1
        LV Size 1.00 GiB
        Mapped size 97.11%
        Current LE 256
        Segments 1
        Allocation inherit
        Read ahead sectors auto
      • currently set to 8192
        Block device 253:4

       

      Attachments

        Issue Links

          Activity

            People

              Unassigned Unassigned
              SolutionGuy Abbas Ahmed
              Votes:
              0 Vote for this issue
              Watchers:
              5 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved:

                Time Tracking

                  Estimated:
                  Original Estimate - Not Specified
                  Not Specified
                  Remaining:
                  Remaining Estimate - Not Specified
                  Not Specified
                  Logged:
                  Time Spent - 1 day, 30 minutes
                  1d 30m

                  Smart Checklist