feat: deploy debian VM using terraform and libvirt, configure it using

ansible with basic services
This commit is contained in:
Aravinth Manivannan 2022-12-19 02:33:15 +05:30
commit b46e41da0b
Signed by: realaravinth
GPG key ID: AD9F0F08E855ED88
16 changed files with 705 additions and 0 deletions

14
debian/.gitignore vendored Normal file
View file

@ -0,0 +1,14 @@
plan
.terraform/
.pytest_cache/
venv/
__pycache__/
ansible/inventory/hosts.ini
plan
/venv/
# Compiled files
*.tfstate
*.tfstate.backup
*.tfstate.lock.info

60
debian/.terraform.lock.hcl vendored Normal file
View file

@ -0,0 +1,60 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/dmacvicar/libvirt" {
version = "0.7.0"
constraints = "~> 0.7.0"
hashes = [
"h1:1RiomFBEdgi6GvqaAf16mW6bRqrxAV0P47HKKwETH3E=",
"zh:1161bfcac075d5790c9b36145811d95241622636b859222f995888471ba04efa",
"zh:317bca5edd36e2497e0ac2ed07dd5e7f09cfd5561a96607cb53fe1af6b0632c0",
"zh:4fa67e3baf6845148f2b4e617fb01c47f0971ce2d945efa805ac5c3820bb0ca6",
"zh:6e17f5f24373e21c0ff463d36d9caa4f08528e13764c5d1d7eceb719dcef6a14",
"zh:84622e2aca8bc91d71d3596fcd1b298c5dfe572c8722ab98084495d26b5c5e7d",
"zh:8ce125d872b26ce9b71a729437eb8ab36944a86da3784edaab7368af43ca3858",
"zh:8fc7eee76776d515c023d013c018a7b9816f0e840578af01bfaf58e49f020c03",
"zh:a4d6fccc0188746be35488396c431e4b313cd1221df408871c710d3a7382b02e",
"zh:b575bb2d2f8987043aecbb22ac3bbf1e9c8b9da49b201b6b225baf2b4595dae4",
"zh:b65b1733c29a09491912a98a829b19c9842af5971fbb358bc0e979b95bf33248",
"zh:b8266ed7b4bce4791fee5433d102d89187974a273574d69f637cfdeb913462c2",
"zh:bd0b842d6f694c6d558d3329a2c157dd9d84074d618d5ced891ef36798b1c97b",
"zh:dacf0299c2c11d84bdaa2f614ca14aeac36ffba0f20dff5a63437a81a61f6867",
"zh:e8c92794a06df42c15ff071859e99c6e95e93dcb40797c4128d31d3a47a27923",
]
}
provider "registry.terraform.io/hashicorp/local" {
version = "2.2.3"
hashes = [
"h1:aWp5iSUxBGgPv1UnV5yag9Pb0N+U1I0sZb38AXBFO8A=",
"zh:04f0978bb3e052707b8e82e46780c371ac1c66b689b4a23bbc2f58865ab7d5c0",
"zh:6484f1b3e9e3771eb7cc8e8bab8b35f939a55d550b3f4fb2ab141a24269ee6aa",
"zh:78a56d59a013cb0f7eb1c92815d6eb5cf07f8b5f0ae20b96d049e73db915b238",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:8aa9950f4c4db37239bcb62e19910c49e47043f6c8587e5b0396619923657797",
"zh:996beea85f9084a725ff0e6473a4594deb5266727c5f56e9c1c7c62ded6addbb",
"zh:9a7ef7a21f48fabfd145b2e2a4240ca57517ad155017e86a30860d7c0c109de3",
"zh:a63e70ac052aa25120113bcddd50c1f3cfe61f681a93a50cea5595a4b2cc3e1c",
"zh:a6e8d46f94108e049ad85dbed60354236dc0b9b5ec8eabe01c4580280a43d3b8",
"zh:bb112ce7efbfcfa0e65ed97fa245ef348e0fd5bfa5a7e4ab2091a9bd469f0a9e",
"zh:d7bec0da5c094c6955efed100f3fe22fca8866859f87c025be1760feb174d6d9",
"zh:fb9f271b72094d07cef8154cd3d50e9aa818a0ea39130bc193132ad7b23076fd",
]
}
provider "registry.terraform.io/hashicorp/template" {
version = "2.2.0"
hashes = [
"h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=",
"zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386",
"zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53",
"zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603",
"zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16",
"zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776",
"zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451",
"zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae",
"zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde",
"zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d",
"zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2",
]
}

38
debian/Makefile vendored Normal file
View file

@ -0,0 +1,38 @@
default:
terraform plan --out=plan
terraform apply plan
inventory: ## Deploy server
terraform plan --out=plan
terraform apply plan
configure: ## Configure server
ansible-playbook -i ./ansible/inventory/hosts.ini -f 10 ./ansible/playbook.yml
ansible-check: ## Check Ansible playbooks
ansible-playbook --check ./ansible/playbook.yml
lint: ## Lint source code
terraform fmt
ansible-lint --write ./ansible/playbook.yml
ansible-lint --write ./ansible/shutdown.yml
. ./venv/bin/activate && black tests/
on: ## Power on VMs
./scripts/on.sh
virsh list
shutdown: ## Shut down vms
ansible-playbook -i ./ansible/inventory/hosts.ini -f 10 ./ansible/shutdown.yml
test: ## Test VM configuration
. ./venv/bin/activate && \
cd tests/ && \
py.test --hosts='ansible://all' \
-n 10 \
--verbose \
--ansible-inventory='../ansible/inventory/hosts.ini'
help: ## Prints help for targets with comments
@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

158
debian/ansible/.gitignore vendored Normal file
View file

@ -0,0 +1,158 @@
inventory/
.env
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
keys
htmlcov/
tmp/
static/

19
debian/ansible/init.sh vendored Executable file
View file

@ -0,0 +1,19 @@
#!/bin/bash
ansible live \
-m ansible.builtin.ping \
-i ./ansible/inventory
ansible live \
-m ansible.builtin.apt \
-f 10 \
-a "update_cache=yes upgrade=safe" \
-i ./ansible/inventory/
ansible live \
-m ansible.builtin.apt \
-f 10 \
-a "name=nginx,git,curl,wget,vim,zip,nginx" \
-i ./ansible/inventory/

117
debian/ansible/playbook.yml vendored Normal file
View file

@ -0,0 +1,117 @@
---
- name: Configure webservers
hosts: debainbasic
remote_user: root
tasks:
- name: Ensure all VMs are reachable
ansible.builtin.ping:
- name: Update package cache
ansible.builtin.apt:
update_cache: true
upgrade: safe
- name: Install git, zip, nginx, wget, curl & other utils
ansible.builtin.apt:
update_cache: true
pkg:
- git
- nginx
- wget
- curl
- gpg
- ca-certificates
- zip
- python3-pip
- virtualenv
- ufw
- fail2ban
- nginx
- dnsutils
- bind9
- python3-setuptools
- name: Create /etc/apt/keyrings dir
ansible.builtin.file:
path: /etc/apt/keyrings
state: directory
recurse: true
- name: Add Docker GPG apt Key
ansible.builtin.apt_key:
url: https://download.docker.com/linux/debian/gpg
state: present
- name: Add Docker Repository
ansible.builtin.apt_repository:
repo: deb https://download.docker.com/linux/debian buster stable
state: present
- name: Update apt and install docker-ce
ansible.builtin.apt:
name: docker-ce
update_cache: true
- name: Install Docker Module for Python
ansible.builtin.pip:
name: docker
- name: Set logging
community.general.ufw:
logging: "on"
- name: Allow port 22 and enable UFW
community.general.ufw:
state: enabled
rule: allow
proto: tcp
port: "22"
- name: Allow port 80
community.general.ufw:
state: enabled
proto: tcp
rule: allow
port: "80"
- name: Allow port 443
community.general.ufw:
state: enabled
proto: tcp
rule: allow
port: "443"
- name: Allow port 53
community.general.ufw:
state: enabled
proto: udp
rule: allow
port: "43"
- name: Enable and start ufw service
ansible.builtin.service:
name: ufw
enabled: true
state: started
- name: Enable and start nginx service
ansible.builtin.service:
name: nginx
enabled: true
state: started
- name: Enable and start bind9
ansible.builtin.service:
name: bind9
enabled: true
state:
started
# - debug: var=ansible_all_ipv4_addresses
# - debug: var=ansible_default_ipv4.address
handlers:
- name: Restart bind9
ansible.builtin.service:
name: nginx
state: restarted

8
debian/ansible/shutdown.yml vendored Normal file
View file

@ -0,0 +1,8 @@
---
- name: Shutdown machines
hosts: debainbasic
remote_user: root
tasks:
- name: Ensure all VMs are reachable
community.general.shutdown:

16
debian/cloud_init.cfg vendored Normal file
View file

@ -0,0 +1,16 @@
#cloud-config
# vim: syntax=yaml
users:
- name: root
ssh_authorized_keys:
- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC/wXdHpwpY/4ubhYTmuNdGepQpj1kchvTUTApxMZyfyVW4uzrPRTYsle1y9QbTBV35qLkNajRC/wmC5/xPchdXpsJpuD9st1HMhLeR8qwaPyptiYJYT+z/WisWw2k6oWhG3QKvPoRtBdW9nhZnkG+O6zkuGXiRHpS7j2VVboDPpWEe1UdELQFVCwfraRal2g3ENFZ/9V1UrW/4ahRnQnSxERplZUm/fgSxQtmXubTkW68ut7yasBsrKFffMm8JztW0tWgTlTKONd3LCjv4juM0t5+cJDotNDnUR86Tq2PG8io7no/h8BWtazmjdpfGgn02ibX26BkdU0LDEYbJt5q9/Fh9TGk2ZwcMQeyepO1AWQgkmHXZWZELqu6MLQpqdtsOjHp9k0MeSpuIbdwzgf10Ydy7vK1z8irS24tVNNnJaMBwOlVOPwfyztHRADPkFcv2lKSjS1uyKR0FIkV8Kvs4txaIjmwv2LfMg6lF5W6j3ZPLyeE4cplJP0DDjzorSanu31xVnqVb3A8V9awsJ/4H7d59bI99c7QHL4K3fBVP3O0gqd31xAVRsdGs5Tj2P+RpiI6o5JJiOa1+DuBdWzrVIXYchQ30ZjaJp1wTNsYLmAsjeYuQZE2tf1xvywdzD4MB4avugDEWikzRWN9V5PHDZr1bamTCCjOrb2PRCd7eSQ== aravinth7820@gmail.com
- name: atm
gecos: Aravinth Manivannan
groups: users, admin
sudo: ALL=(ALL) NOPASSWD:ALL
shell: /bin/bash
lock_passwd: true
plain_text_passwd: fooabr12
ssh_authorized_keys:
- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC/wXdHpwpY/4ubhYTmuNdGepQpj1kchvTUTApxMZyfyVW4uzrPRTYsle1y9QbTBV35qLkNajRC/wmC5/xPchdXpsJpuD9st1HMhLeR8qwaPyptiYJYT+z/WisWw2k6oWhG3QKvPoRtBdW9nhZnkG+O6zkuGXiRHpS7j2VVboDPpWEe1UdELQFVCwfraRal2g3ENFZ/9V1UrW/4ahRnQnSxERplZUm/fgSxQtmXubTkW68ut7yasBsrKFffMm8JztW0tWgTlTKONd3LCjv4juM0t5+cJDotNDnUR86Tq2PG8io7no/h8BWtazmjdpfGgn02ibX26BkdU0LDEYbJt5q9/Fh9TGk2ZwcMQeyepO1AWQgkmHXZWZELqu6MLQpqdtsOjHp9k0MeSpuIbdwzgf10Ydy7vK1z8irS24tVNNnJaMBwOlVOPwfyztHRADPkFcv2lKSjS1uyKR0FIkV8Kvs4txaIjmwv2LfMg6lF5W6j3ZPLyeE4cplJP0DDjzorSanu31xVnqVb3A8V9awsJ/4H7d59bI99c7QHL4K3fBVP3O0gqd31xAVRsdGs5Tj2P+RpiI6o5JJiOa1+DuBdWzrVIXYchQ30ZjaJp1wTNsYLmAsjeYuQZE2tf1xvywdzD4MB4avugDEWikzRWN9V5PHDZr1bamTCCjOrb2PRCd7eSQ== aravinth7820@gmail.com

121
debian/conf.tf vendored Normal file
View file

@ -0,0 +1,121 @@
terraform {
required_version = ">= 0.13"
required_providers {
libvirt = {
source = "dmacvicar/libvirt"
version = "~> 0.7.0"
}
}
}
# instance the provider
provider "libvirt" {
uri = "qemu:///system"
}
resource "libvirt_pool" "debian_basic" {
name = "debian_basic"
type = "dir"
path = "/home/atm/code/libvirt/pool/debian_basic"
}
resource "libvirt_volume" "debian11-qcow2" {
name = "debian11-qcow2"
pool = libvirt_pool.debian_basic.name
source = "https://cloud.debian.org/images/cloud/bullseye/latest/debian-11-genericcloud-amd64.qcow2"
format = "qcow2"
}
variable "vm_count" {
default = 1
}
resource "libvirt_volume" "domain_debian_basic_volume" {
name = "domain_debian_basic_volume-${count.index}"
base_volume_id = libvirt_volume.debian11-qcow2.id
count = var.vm_count
size = 5368709120
}
data "template_file" "user_data" {
template = file("${path.module}/cloud_init.cfg")
}
data "template_file" "network_config" {
template = file("${path.module}/network_config.cfg")
}
resource "libvirt_cloudinit_disk" "commoninit" {
name = "commoninit.iso"
user_data = data.template_file.user_data.rendered
network_config = data.template_file.network_config.rendered
pool = libvirt_pool.debian_basic.name
}
# Create the machine
resource "libvirt_domain" "domain_debian_basic" {
count = var.vm_count
name = "debian_basic_${count.index}"
memory = "3000"
vcpu = 4
cloudinit = libvirt_cloudinit_disk.commoninit.id
network_interface {
network_name = "default"
wait_for_lease = true
}
# IMPORTANT: this is a known bug on cloud images, since they expect a console
# we need to pass it
# https://bugs.launchpad.net/cloud-images/+bug/1573095
console {
type = "pty"
target_port = "0"
target_type = "serial"
}
console {
type = "pty"
target_type = "virtio"
target_port = "1"
}
disk {
volume_id = element(libvirt_volume.domain_debian_basic_volume.*.id, count.index)
}
graphics {
type = "spice"
listen_type = "address"
autoport = true
}
}
locals {
vm_ips = [for i in libvirt_domain.domain_debian_basic : i.network_interface.0.addresses[0]]
vm_names = [for i in libvirt_domain.domain_debian_basic : i.name]
vm_map = [for i in libvirt_domain.domain_debian_basic : {
ip = i.network_interface.0.addresses[0],
name = i.name
}]
}
output "debian_ip" {
value = local.vm_map
}
resource "local_file" "hosts_yml" {
content = templatefile("./templates/hosts.yml.tftpl",
{
vm_ips = local.vm_ips,
vm_names = local.vm_names,
vms = local.vm_map
})
filename = "./ansible/inventory/hosts.ini"
}

4
debian/network_config.cfg vendored Normal file
View file

@ -0,0 +1,4 @@
version: 2
ethernets:
ens3:
dhcp4: true

36
debian/requirements.txt vendored Normal file
View file

@ -0,0 +1,36 @@
astroid==2.12.12
attrs==22.1.0
black==22.10.0
certifi==2022.9.24
charset-normalizer==2.1.1
click==8.1.3
dill==0.3.6
exceptiongroup==1.0.0rc9
execnet==1.9.0
greenlet==1.1.3.post0
idna==3.4
iniconfig==1.1.1
isort==5.10.1
jedi==0.18.1
lazy-object-proxy==1.7.1
mccabe==0.7.0
msgpack==1.0.4
mypy-extensions==0.4.3
packaging==21.3
parso==0.8.3
pathspec==0.10.1
platformdirs==2.5.2
pluggy==1.0.0
py==1.11.0
pylint==2.15.5
pynvim==0.4.3
pyparsing==3.0.9
pytest==7.2.0
pytest-forked==1.4.0
pytest-testinfra==6.8.0
pytest-xdist==2.5.0
requests==2.28.1
tomli==2.0.1
tomlkit==0.11.5
urllib3==1.26.12
wrapt==1.14.1

7
debian/scripts/on.sh vendored Executable file
View file

@ -0,0 +1,7 @@
#!/bin/bash
for vm in $(virsh list --all --name --state-shutoff); do \
echo "[*] Starting vm: $vm"; \
virsh start $vm; \
done

4
debian/templates/hosts.yml.tftpl vendored Normal file
View file

@ -0,0 +1,4 @@
[debainbasic]
%{ for vm in vms ~}
${vm.name} ansible_host=${vm.ip} ansible_user=root
%{ endfor ~}

9
debian/tests/requirements.txt vendored Normal file
View file

@ -0,0 +1,9 @@
attrs==22.1.0
iniconfig==1.1.1
packaging==21.3
pluggy==1.0.0
py==1.11.0
pyparsing==3.0.9
pytest==7.1.3
pytest-testinfra==6.8.0
tomli==2.0.1

78
debian/tests/test_basic.py vendored Normal file
View file

@ -0,0 +1,78 @@
import os
def test_packages_are_installed(host):
packages = [
"nginx",
"ufw",
"docker-ce",
"git",
"nginx",
"wget",
"curl",
"gpg",
"dnsutils",
"ca-certificates",
"zip",
"python3-pip",
"virtualenv",
"python3-setuptools",
]
for p in packages:
print(f"[*] Checking if {p} is installed")
pkg = host.package(p)
assert pkg.is_installed
def test_ssh_is_listening(host):
socket = host.socket(f"tcp://0.0.0.0:22")
assert socket.is_listening
def test_docker_is_installed(host):
keyring_dir = host.file("/etc/apt/keyrings")
assert keyring_dir.exists
assert keyring_dir.is_directory
def test_ufw_service_running_and_enabled(host):
service = host.service("ufw")
assert service.is_running
assert service.is_enabled
def test_ssh_service_running_and_enabled(host):
service = host.service("ssh")
assert service.is_running
assert service.is_enabled
def test_nginx_service_running_and_enabled(host):
service = host.service("nginx")
assert service.is_running
assert service.is_enabled
#
#
# def test_fail2ban_is_installed(host):
# pkg = host.package("fail2ban")
# assert pkg.is_installed
#
#
# def test_fail2ban_is_enabled_and_running(host):
# service = host.service("fail2ban")
# assert service.is_running
# assert service.is_enabled
#
#
def test_ssh_is_installed(host):
pkg = host.package("openssh-server")
assert pkg.is_installed
#
# def test_ssh_is_enabled_and_running(host):
# service = host.service("sshd")
# assert service.is_running
# assert service.is_enabled

16
debian/tests/test_dns.py vendored Normal file
View file

@ -0,0 +1,16 @@
import os
def test_packages_are_installed(host):
packages = [
"bind9"
]
for p in packages:
print(f"[*] Checking if {p} is installed")
pkg = host.package(p)
assert pkg.is_installed
def test_nginx_service_running_and_enabled(host):
service = host.service("bind9")
assert service.is_running
assert service.is_enabled