LVM Volumes wrongly sized on new AWS deployment

I deployed a new instance from AWS AMI version 1.17.2.

It stopped working in a few hours. It appears it ran out of disk space. 

Closer examination revealed that there is an incorrect LVM sizing. Also attached is the latest messages syslog.

Details below:

 

[root@ip-172-31-17-184 srv]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
nvme0n1 259:1 0 8G 0 disk
└─nvme0n1p1 259:2 0 8G 0 part /
nvme1n1 259:0 0 100G 0 disk
├─DataVG-ThinPool_tmeta 253:0 0 100M 0 lvm
│ └─DataVG-ThinPool-tpool 253:2 0 99.8G 0 lvm
│ ├─DataVG-ThinPool 253:3 0 99.8G 0 lvm
│ └─DataVG-DataLV 253:4 0 1G 0 lvm /srv
└─DataVG-ThinPool_tdata 253:1 0 99.8G 0 lvm
└─DataVG-ThinPool-tpool 253:2 0 99.8G 0 lvm
├─DataVG-ThinPool 253:3 0 99.8G 0 lvm
└─DataVG-DataLV 253:4 0 1G 0 lvm /srv

 

[root@ip-172-31-17-184 srv]# mount
sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime,seclabel)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
devtmpfs on /dev type devtmpfs (rw,nosuid,seclabel,size=896164k,nr_inodes=224041,mode=755)
securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev,seclabel)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,seclabel,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,nodev,seclabel,mode=755)
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,seclabel,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd)
pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime)
cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,net_prio,net_cls)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,cpuacct,cpu)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,freezer)
cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,pids)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,cpuset)
cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,memory)
cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,perf_event)
cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,hugetlb)
cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,blkio)
cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,devices)
configfs on /sys/kernel/config type configfs (rw,relatime)
/dev/nvme0n1p1 on / type xfs (rw,relatime,seclabel,attr2,inode64,noquota)
rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime)
selinuxfs on /sys/fs/selinux type selinuxfs (rw,relatime)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime,seclabel)
debugfs on /sys/kernel/debug type debugfs (rw,relatime)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=11025)
mqueue on /dev/mqueue type mqueue (rw,relatime,seclabel)
/dev/mapper/DataVG-DataLV on /srv type xfs (rw,relatime,seclabel,attr2,inode64,logbsize=64k,sunit=128,swidth=128,noquota)
tmpfs on /run/user/1001 type tmpfs (rw,nosuid,nodev,relatime,seclabel,size=184316k,mode=700,uid=1001,gid=1001)

 

[root@ip-172-31-17-184 srv]# df -lh
Filesystem Size Used Avail Use% Mounted on
devtmpfs 876M 0 876M 0% /dev
tmpfs 900M 0 900M 0% /dev/shm
tmpfs 900M 118M 783M 14% /run
tmpfs 900M 0 900M 0% /sys/fs/cgroup
/dev/nvme0n1p1 8.0G 3.7G 4.4G 46% /
/dev/mapper/DataVG-DataLV 1014M 1014M 20K 100% /srv
tmpfs 180M 0 180M 0% /run/user/1001

 

[root@ip-172-31-17-184 ~]# lvdisplay
— Logical volume —
LV Name ThinPool
VG Name DataVG
LV UUID 725l1r-f3go-vL86-eQNd-EHIS-44DT-NlwHq2
LV Write Access read/write
LV Creation host, time ip-172-30-5-209.ec2.internal, 2019-10-24 13:42:01 +0000
LV Pool metadata ThinPool_tmeta
LV Pool data ThinPool_tdata
LV Status available

1. open 2
   LV Size 99.80 GiB
   Allocated pool data 0.97%
   Allocated metadata 10.91%
   Current LE 25549
   Segments 1
   Allocation inherit
   Read ahead sectors auto

• currently set to 8192
  Block device 253:2

— Logical volume —
LV Path /dev/DataVG/DataLV
LV Name DataLV
VG Name DataVG
LV UUID 5ctyw7-01bK-Obu8-iQMc-a0fM-gIYo-9iF6MW
LV Write Access read/write
LV Creation host, time ip-172-30-5-209.ec2.internal, 2019-10-24 13:42:03 +0000
LV Pool name ThinPool
LV Status available

1. open 1
   LV Size 1.00 GiB
   Mapped size 97.11%
   Current LE 256
   Segments 1
   Allocation inherit
   Read ahead sectors auto

• currently set to 8192
  Block device 253:4

 

[root@ip-172-31-17-184 ~]# vgdisplay
— Volume group —
VG Name DataVG
System ID
Format lvm2
Metadata Areas 1
Metadata Sequence No 657
VG Access read/write
VG Status resizable
MAX LV 0
Cur LV 2
Open LV 1
Max PV 0
Cur PV 1
Act PV 1
VG Size <100.00 GiB
PE Size 4.00 MiB
Total PE 25599
Alloc PE / Size 25599 / <100.00 GiB
Free PE / Size 0 / 0
VG UUID s9hIMr-q4km-K14c-Z2oD-41nE-2NRF-Rc8tuj

 

[root@ip-172-31-17-184 ~]# lvdisplay
— Logical volume —
LV Name ThinPool
VG Name DataVG
LV UUID 725l1r-f3go-vL86-eQNd-EHIS-44DT-NlwHq2
LV Write Access read/write
LV Creation host, time ip-172-30-5-209.ec2.internal, 2019-10-24 13:42:01 +0000
LV Pool metadata ThinPool_tmeta
LV Pool data ThinPool_tdata
LV Status available

1. open 2
   LV Size 99.80 GiB
   Allocated pool data 0.97%
   Allocated metadata 10.91%
   Current LE 25549
   Segments 1
   Allocation inherit
   Read ahead sectors auto

• currently set to 8192
  Block device 253:2

— Logical volume —
LV Path /dev/DataVG/DataLV
LV Name DataLV
VG Name DataVG
LV UUID 5ctyw7-01bK-Obu8-iQMc-a0fM-gIYo-9iF6MW
LV Write Access read/write
LV Creation host, time ip-172-30-5-209.ec2.internal, 2019-10-24 13:42:03 +0000
LV Pool name ThinPool
LV Status available

1. open 1
   LV Size 1.00 GiB
   Mapped size 97.11%
   Current LE 256
   Segments 1
   Allocation inherit
   Read ahead sectors auto

• currently set to 8192
  Block device 253:4