wip-cache #2

Merged
realaravinth merged 6 commits from wip-cache into master 2023-12-09 02:58:16 +05:30
21 changed files with 466 additions and 1 deletions

View file

@ -19,6 +19,12 @@ define configure_locust
ansible-playbook -i $(INVENTORY) -f 10 ./ansible/locust.yml ansible-playbook -i $(INVENTORY) -f 10 ./ansible/locust.yml
endef endef
define configure_cache
. ./venv/bin/activate && \
ansible-playbook -i $(INVENTORY) -f 10 ./ansible/cache.yml
endef
define test_base define test_base
. ./venv/bin/activate && \ . ./venv/bin/activate && \
cd tests/ && \ cd tests/ && \
@ -31,6 +37,19 @@ define test_base
endef endef
define test_cache
. ./venv/bin/activate && \
cd tests/cache/ && \
ANSIBLE_REMOTE_USER=root \
py.test --hosts='ansible://mcaptcha_hosts' \
-n 10 \
--verbose \
--ansible-inventory="../../${INVENTORY}" \
base.py
endef
define test_locust define test_locust
. ./venv/bin/activate && \ . ./venv/bin/activate && \
cd tests/locust/ && \ cd tests/locust/ && \
@ -62,6 +81,13 @@ lint: ## Lint source code
conf.dos: ## Configure all DoS VMs conf.dos: ## Configure all DoS VMs
$(call configure_locust) $(call configure_locust)
# ```bash
# INVENTORY=./terraform/dos/hosts.ini make conf.cache
# ```
conf.cache: ## Configure all cache VMs
$(call configure_cache)
# ```bash # ```bash
# INVENTORY=./terraform/dos/hosts.ini make conf.ping # INVENTORY=./terraform/dos/hosts.ini make conf.ping
# ``` # ```
@ -76,5 +102,10 @@ test: ## Run all tests
test.base: ## Test base configuration on all VMs test.base: ## Test base configuration on all VMs
$(call test_base) $(call test_base)
test.cache: ## Test cache configuration
$(call test_cache)
help: ## Prints help for targets with comments help: ## Prints help for targets with comments
@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-].+:.*?## .*$$' | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-].+:.*?## .*$$' | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

View file

@ -17,6 +17,7 @@
become: true become: true
ansible.builtin.apt: ansible.builtin.apt:
update_cache: true update_cache: true
cache_valid_time: 3600
pkg: pkg:
- git - git
- wget - wget

15
ansible/cache.yml Normal file
View file

@ -0,0 +1,15 @@
# SPDX-FileCopyrightText: 2023 Aravinth Manivannan <realaravinth@batsense.net>
#
# SPDX-License-Identifier: AGPL-3.0-or-later
---
- name: Base configuration
ansible.builtin.import_playbook: base.yml
- name: Install redis cache
hosts: mcaptcha_hosts
remote_user: atm
pre_tasks:
- name: Ensure all VMs are reachable
ansible.builtin.ping:
roles:
- cache

7
ansible/roles/cache/handlers/main.yml vendored Normal file
View file

@ -0,0 +1,7 @@
- name: Restart redis
listen: restart redis
become: true
ansible.builtin.service:
name: redis
enabled: true
state: restarted

86
ansible/roles/cache/tasks/main.yml vendored Normal file
View file

@ -0,0 +1,86 @@
# SPDX-FileCopyrightText: 2023 Aravinth Manivannan <realaravinth@batsense.net>
#
# SPDX-License-Identifier: AGPL-3.0-or-later
---
- name: Install redis-server
become: true
ansible.builtin.apt:
update_cache: true
cache_valid_time: 3600
pkg:
- redis-server
- name: Create Redis plugins dir
become: true
notify: restart redis
ansible.builtin.file:
path: /usr/lib/redis/modules
owner: redis
group: redis
state: directory
mode: "0755"
- name: Load mCaptcha module
become: true
notify: restart redis
ansible.builtin.lineinfile:
dest: /etc/redis/redis.conf
line: "{{ item }}"
loop:
- "loadmodule /usr/lib/redis/modules/libcache.so"
- "bind 0.0.0.0"
- name: Copy custom Redis configuration
become: true
notify: restart redis
ansible.builtin.file:
path: "{{ item }}"
owner: redis
group: redis
force: true
loop:
- /etc/redis/redis.conf
- /etc/redis/
- name: Create download dir
ansible.builtin.file:
path: /tmp/cache-lib
state: directory
mode: "0755"
- name: Download plugin
ansible.builtin.get_url:
url: https://dl.mcaptcha.org/mcaptcha/cache/master/cache-master-linux-amd64.tar.gz
dest: /tmp/cache-lib/
checksum: sha256:https://dl.mcaptcha.org/mcaptcha/cache/master/cache-master-linux-amd64.tar.gz.sha256
- name: Extract cache-master-linux-amd64.tar.gz into /var/lib/foo
ansible.builtin.unarchive:
src: /tmp/cache-lib/cache-master-linux-amd64.tar.gz
remote_src: true
dest: /tmp/cache-lib/
- name: Copy custom Redis configuration
become: true
notify: restart redis
ansible.builtin.copy:
src: /tmp/cache-lib/cache-master-linux-amd64/libcache.so
remote_src: true
dest: /usr/lib/redis/modules/
owner: redis
group: redis
force: true
mode: "0755"
# - name: Delete download dir
# ansible.builtin.file:
# path: /tmp/cache-lib
# state: absent
- name: Allow port 6379 for redis
become: true
community.general.ufw:
state: enabled
rule: allow
proto: tcp
port: "6379"

View file

@ -7,5 +7,6 @@
become: true become: true
ansible.builtin.apt: ansible.builtin.apt:
update_cache: true update_cache: true
cache_valid_time: 3600
pkg: pkg:
- docker-compose - docker-compose

View file

@ -7,6 +7,7 @@
become: true become: true
ansible.builtin.apt: ansible.builtin.apt:
update_cache: true update_cache: true
cache_valid_time: 3600
pkg: pkg:
- ntp - ntp

View file

@ -7,6 +7,7 @@
become: true become: true
ansible.builtin.apt: ansible.builtin.apt:
update_cache: true update_cache: true
cache_valid_time: 3600
pkg: pkg:
- ufw - ufw

45
terraform/mcaptcha/.gitignore vendored Normal file
View file

@ -0,0 +1,45 @@
# Compiled files
*.tfstate
*.tfstate.backup
*.tfstate.lock.info
# logs
*.log
# Directories
.terraform/
.vagrant/
# SSH Keys
*.pem
# Backup files
*.bak
# Ignored Terraform files
*gitignore*.tf
# Ignore Mac .DS_Store files
.DS_Store
# Ignored vscode files
.vscode/
# Ignore Any Generated JSON Files
operations/automation-script/apply.json
operations/automation-script/configversion.json
operations/automation-script/run.template.json
operations/automation-script/run.json
operations/automation-script/variable.template.json
operations/automation-script/variable.json
operations/automation-script/workspace.template.json
operations/automation-script/workspace.json
operations/sentinel-policies-scripts/create-policy.template.json
operations/sentinel-policies-scripts/create-policy.json
operations/variable-scripts/variable.template.json
operations/variable-scripts/variable.json
# Sentinel runtime directory
.sentinel
dos
hosts.ini

View file

@ -0,0 +1,53 @@
# This file is maintained automatically by "tofu init".
# Manual edits may be lost in future updates.
provider "registry.opentofu.org/dmacvicar/libvirt" {
version = "0.7.6"
constraints = "~> 0.7.0"
hashes = [
"h1:mmbm4vTyC/DCGO4Ed/vbp5AKvy1gmVn/94fzB9VmR08=",
"zh:0bde54f6f658b20b620b875daf106b5b25b1bae4d15408d6c5f06d58360e254d",
"zh:0c97c6930015918b8a34b6d7a2b0c3d17a649c226fcd1874fcba5bbbc0f35972",
"zh:1bdd7aa0011c5f024a09a124836ee9bc8e71b05a6ece810c61824275fd3f695f",
"zh:2b0cc7c794e4caf395d84ffff0b380d17e4b3219a4696264271bfe5059450efe",
"zh:2f8633f7fe07f76c188836ed6f93321ec5fbf5c004bc7699e1741d9b21ed5f37",
"zh:5bf47eed286ce55ed10a5cf657de49a34ab21cc8677c56fef3aab69cdde41a27",
"zh:7dca790fc5fd1d42bc4bc7170be003a7093602026d0f95c8aab84ad551fdf2a4",
"zh:80476b68bc84e3d661d1390025f83879b88f9cdc836de9751af09bd5716089cb",
"zh:82f3e2f3f50176cd6041c8ba36e295cbda1b289ef52ab75b5eceb0f921f64f7b",
"zh:a179b165f3b9bb9a67ebbbf9d73157ded33f02d476b2f58906389dca03b653c9",
"zh:acae54a5d0616f22b3180ddd8e8aad39af664e604394fdacf1f7b337bca2d5b4",
"zh:da4406a2428a9a7e98272c032cb93431c3919253af2fe9934b532d26c0deab09",
"zh:f63dbd8e579ab5268d01ffab4503b8a8e736b70d1a04e4f271559ba8dd133dcd",
"zh:f85c1d9e51a94ecde137435c9d6b0fb7be590437ea8a725334d1577eebbc550c",
]
}
provider "registry.opentofu.org/hashicorp/local" {
version = "2.4.0"
hashes = [
"h1:pWJMQ+uRtVtHg97vU2zSCuYcZTuDQ7FJz+QanfSGMXM=",
"zh:184d6ec1f0e77713b37f0d9cf943b1371f2aa2f44c2c5a618978e897ce3dccab",
"zh:2205a7955a4051c2c25e69646a60746d9416b73001491808ae5d10620f7b7ac1",
"zh:256ddc56457f725819dc6be62f2d0bb3b9fee40a61771317bb32353df5b5c1a0",
"zh:70146e603f540523f6fa2251dd52c225db5a92bda8c07fd198ed51ae2b50176b",
"zh:8c3f9fe12ab8843e25ff7edabc26e01df4a0e8db204e432600a4c77a95ec0535",
"zh:b003e421f643d14247d31dcb7f0f6470c46f772d0e15a175a555a525ce344bf2",
"zh:b4c8ad7c5696aeb2a52adf6047d1e01943fafa57dc123d5192542527406ffd72",
"zh:c3b6fbfa431f3c085621c74596ee63681a278fd433a4758f33c627e8936d5cb3",
"zh:d2e57b19295b326d84ca5f39b797849d901170d5509aa7558f2a6545c9ce72a9",
"zh:e2307421b0b380eb0e8fcee008e0af98ae30fccbfc9e9a1d24d952489e9b0df9",
]
}
provider "registry.opentofu.org/hashicorp/template" {
version = "2.2.0"
hashes = [
"h1:tdS0otiAtvUV8uLJWJNfcqOPo3llj7FyRzExw6X1srY=",
"zh:374c28bafc43cd65e578cb209efc9eee4c1cec7618f451528e928db98059e8c8",
"zh:6a2982e70fbc2ab2668d624c648ef2eb32243c1a1185246b03991a7a21326db9",
"zh:af83169c21bb13f141510a349e1f70cf7d893247a269bd71cad74dd22f1df0f5",
"zh:b81a5bedc91a1a81b938c393247248d6c3d1bd8ea685541f9c858908c0afb6b3",
"zh:de15486244af2d29d44d510d647cd6e0b1408e89952261013c572b7c9bfd744b",
]
}

View file

@ -0,0 +1,23 @@
#cloud-config
# vim: syntax=yaml
users:
- name: root
ssh_authorized_keys:
- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC/wXdHpwpY/4ubhYTmuNdGepQpj1kchvTUTApxMZyfyVW4uzrPRTYsle1y9QbTBV35qLkNajRC/wmC5/xPchdXpsJpuD9st1HMhLeR8qwaPyptiYJYT+z/WisWw2k6oWhG3QKvPoRtBdW9nhZnkG+O6zkuGXiRHpS7j2VVboDPpWEe1UdELQFVCwfraRal2g3ENFZ/9V1UrW/4ahRnQnSxERplZUm/fgSxQtmXubTkW68ut7yasBsrKFffMm8JztW0tWgTlTKONd3LCjv4juM0t5+cJDotNDnUR86Tq2PG8io7no/h8BWtazmjdpfGgn02ibX26BkdU0LDEYbJt5q9/Fh9TGk2ZwcMQeyepO1AWQgkmHXZWZELqu6MLQpqdtsOjHp9k0MeSpuIbdwzgf10Ydy7vK1z8irS24tVNNnJaMBwOlVOPwfyztHRADPkFcv2lKSjS1uyKR0FIkV8Kvs4txaIjmwv2LfMg6lF5W6j3ZPLyeE4cplJP0DDjzorSanu31xVnqVb3A8V9awsJ/4H7d59bI99c7QHL4K3fBVP3O0gqd31xAVRsdGs5Tj2P+RpiI6o5JJiOa1+DuBdWzrVIXYchQ30ZjaJp1wTNsYLmAsjeYuQZE2tf1xvywdzD4MB4avugDEWikzRWN9V5PHDZr1bamTCCjOrb2PRCd7eSQ== aravinth7820@gmail.com
- name: atm
gecos: Aravinth Manivannan
groups: users, admin
sudo: ALL=(ALL) NOPASSWD:ALL
shell: /bin/bash
lock_passwd: true
plain_text_passwd: fooabr12
ssh_authorized_keys:
- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC/wXdHpwpY/4ubhYTmuNdGepQpj1kchvTUTApxMZyfyVW4uzrPRTYsle1y9QbTBV35qLkNajRC/wmC5/xPchdXpsJpuD9st1HMhLeR8qwaPyptiYJYT+z/WisWw2k6oWhG3QKvPoRtBdW9nhZnkG+O6zkuGXiRHpS7j2VVboDPpWEe1UdELQFVCwfraRal2g3ENFZ/9V1UrW/4ahRnQnSxERplZUm/fgSxQtmXubTkW68ut7yasBsrKFffMm8JztW0tWgTlTKONd3LCjv4juM0t5+cJDotNDnUR86Tq2PG8io7no/h8BWtazmjdpfGgn02ibX26BkdU0LDEYbJt5q9/Fh9TGk2ZwcMQeyepO1AWQgkmHXZWZELqu6MLQpqdtsOjHp9k0MeSpuIbdwzgf10Ydy7vK1z8irS24tVNNnJaMBwOlVOPwfyztHRADPkFcv2lKSjS1uyKR0FIkV8Kvs4txaIjmwv2LfMg6lF5W6j3ZPLyeE4cplJP0DDjzorSanu31xVnqVb3A8V9awsJ/4H7d59bI99c7QHL4K3fBVP3O0gqd31xAVRsdGs5Tj2P+RpiI6o5JJiOa1+DuBdWzrVIXYchQ30ZjaJp1wTNsYLmAsjeYuQZE2tf1xvywdzD4MB4avugDEWikzRWN9V5PHDZr1bamTCCjOrb2PRCd7eSQ== aravinth7820@gmail.com
ssh_pwauth: true
chpasswd:
list: |
root:foobar12
atm:foobar12
expire: False

View file

@ -0,0 +1,46 @@
# SPDX-FileCopyrightText: 2023 Aravinth Manivannan <realaravinth@batsense.net>
#
# SPDX-License-Identifier: AGPL-3.0-or-later
terraform {
required_version = ">= 0.13"
required_providers {
libvirt = {
source = "dmacvicar/libvirt"
version = "~> 0.7.0"
}
}
}
provider "libvirt" {
uri = var.libvirt_uri
}
resource "libvirt_pool" "mcaptcha_basic" {
name = "mcaptcha_basic"
type = "dir"
path = var.libvirt_pool_path
}
resource "libvirt_volume" "debian-mcaptcha-qcow2" {
name = "debian-mcaptcha-qcow2"
pool = libvirt_pool.mcaptcha_basic.name
source = var.libvirt_debian_src
format = "qcow2"
}
data "template_file" "user_data" {
template = file("${path.module}/cloud_init.cfg")
}
data "template_file" "network_config" {
template = file("${path.module}/network_config.cfg")
}
resource "libvirt_cloudinit_disk" "commoninit" {
name = "commoninit.iso"
user_data = data.template_file.user_data.rendered
network_config = data.template_file.network_config.rendered
pool = libvirt_pool.mcaptcha_basic.name
}

BIN
terraform/mcaptcha/mcaptcha Normal file

Binary file not shown.

View file

@ -0,0 +1,52 @@
# SPDX-FileCopyrightText: 2023 Aravinth Manivannan <realaravinth@batsense.net>
#
# SPDX-License-Identifier: AGPL-3.0-or-later
resource "libvirt_volume" "mcaptcha_volume" {
name = "mcaptcha_volume-${count.index}"
base_volume_id = libvirt_volume.debian-mcaptcha-qcow2.id
count = var.mcaptcha_vm_count
pool = libvirt_pool.mcaptcha_basic.name
size = var.mcaptcha_vm_disk_size
}
resource "libvirt_domain" "mcaptcha_mcaptcha" {
count = var.mcaptcha_vm_count
name = "mcaptcha_mcaptcha_${count.index}"
memory = var.mcaptcha_vm_memory
vcpu = var.mcaptcha_vm_vcpu
cloudinit = libvirt_cloudinit_disk.commoninit.id
console {
type = "pty"
target_port = "0"
target_type = "serial"
}
console {
type = "pty"
target_type = "virtio"
target_port = "1"
}
network_interface {
network_name = "default"
wait_for_lease = true
}
disk {
volume_id = element(libvirt_volume.mcaptcha_volume.*.id, count.index)
}
}
locals {
mcaptcha_vm_ips = [for i in libvirt_domain.mcaptcha_mcaptcha : i.network_interface.0.addresses[0]]
mcaptcha_vm_names = [for i in libvirt_domain.mcaptcha_mcaptcha : i.name]
mcaptcha_vm_map = [for i in libvirt_domain.mcaptcha_mcaptcha : {
ip = i.network_interface.0.addresses[0],
name = i.name
}]
}

View file

@ -0,0 +1,4 @@
version: 2
ethernets:
ens3:
dhcp4: true

View file

@ -0,0 +1,20 @@
# SPDX-FileCopyrightText: 2023 Aravinth Manivannan <realaravinth@batsense.net>
#
# SPDX-License-Identifier: AGPL-3.0-or-later
output "mcaptcha_mcaptcha_ip" {
value = local.mcaptcha_vm_map
}
resource "local_file" "hosts_yml" {
content = templatefile("./templates/hosts.yml.tftpl",
{
mcaptcha_vms_ips = local.mcaptcha_vm_ips,
mcaptcha_vms_names = local.mcaptcha_vm_names,
mcaptcha_vms = local.mcaptcha_vm_map
})
filename = "hosts.ini"
}

View file

@ -0,0 +1,7 @@
#!/bin/bash
for vm in $(virsh list --all --name --state-shutoff); do \
echo "[*] Starting vm: $vm"; \
virsh start $vm; \
done

View file

@ -0,0 +1,4 @@
[mcaptcha_hosts]
%{ for vm in mcaptcha_vms ~}
${vm.name} ansible_host=${vm.ip} ansible_user=atm
%{ endfor ~}

View file

@ -0,0 +1,46 @@
# SPDX-FileCopyrightText: 2023 Aravinth Manivannan <realaravinth@batsense.net>
#
# SPDX-License-Identifier: AGPL-3.0-or-later
/* main.tf */
variable "libvirt_uri" {
description = "URI of libvert socket"
type = string
default = "qemu:///system"
}
variable "libvirt_pool_path" {
description = "Path of libvirt storage pool"
type = string
default = "/home/atm/code/libvirt/pool/mcaptcha_basic"
}
variable "libvirt_debian_src" {
description = "Location of Debian 11 qcow2 image"
type = string
default = "/home/atm/disk-images/debian/12/clould/debian-12-generic-amd64.qcow2"
}
/* mcaptcha/mcaptcha server server */
variable "mcaptcha_vm_count" {
description = "Number of VMs to be deployed to run mcaptcha/mcaptcha"
type = number
default = 1
}
variable "mcaptcha_vm_disk_size" {
description = "Size of disk of VMs running mcaptcha/mcaptcha in bytes"
type = number
default = 8000000000 # 8GB
}
variable "mcaptcha_vm_memory" {
description = "Memory of VMs running mcaptcha/mcaptcha in MB"
type = number
default = 2000 # 2GB
}
variable "mcaptcha_vm_vcpu" {
description = "Number of CPUs of VMs running mcaptcha/mcaptcha"
type = number
default = 4
}

22
tests/cache/base.py vendored Normal file
View file

@ -0,0 +1,22 @@
# SPDX-FileCopyrightText: 2023 Aravinth Manivannan <realaravinth@batsense.net>
#
# SPDX-License-Identifier: AGPL-3.0-or-later
import os
def test_redis_is_listening(host):
socket = host.socket(f"tcp://0.0.0.0:6379")
assert socket.is_listening
def test_redis_config_exists(host):
config = host.file("/etc/redis/redis.conf")
assert config.exists
assert config.is_file
def test_redis_service_running_and_enabled(host):
service = host.service("redis")
assert service.is_running
assert service.is_enabled