Merge pull request 'wip-cache' (#2) from wip-cache into master

Reviewed-on: #2

Makefile

@@ -19,6 +19,12 @@ define configure_locust
 	ansible-playbook -i $(INVENTORY) -f 10 ./ansible/locust.yml
 endef
 
+define configure_cache
+	. ./venv/bin/activate &&  \
+	ansible-playbook -i $(INVENTORY) -f 10 ./ansible/cache.yml
+endef
+
+
 define test_base
         . ./venv/bin/activate &&  \
                 cd tests/ && \
@@ -31,6 +37,19 @@ define test_base
 
 endef
 
+define test_cache
+        . ./venv/bin/activate &&  \
+                cd tests/cache/ && \
+				ANSIBLE_REMOTE_USER=root \
+                py.test --hosts='ansible://mcaptcha_hosts' \
+                -n 10 \
+                --verbose \
+				--ansible-inventory="../../${INVENTORY}" \
+                base.py
+
+endef
+
+
 define test_locust
 	. ./venv/bin/activate &&  \
 		cd tests/locust/ && \
@@ -62,6 +81,13 @@ lint: ## Lint source code
 conf.dos: ## Configure all DoS VMs
 	$(call configure_locust)
 
+
+# ```bash
+# INVENTORY=./terraform/dos/hosts.ini make conf.cache
+# ```
+conf.cache: ## Configure all cache VMs
+	$(call configure_cache)
+
 # ```bash
 # INVENTORY=./terraform/dos/hosts.ini make conf.ping
 # ```
@@ -76,5 +102,10 @@ test: ## Run all tests
 test.base: ## Test base configuration on all VMs
 	$(call test_base)
 
+test.cache: ## Test cache configuration
+	$(call test_cache)
+
+
+
 help: ## Prints help for targets with comments
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-].+:.*?## .*$$' | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

ansible/base.yml

@@ -17,6 +17,7 @@
       become: true
       ansible.builtin.apt:
         update_cache: true
+        cache_valid_time: 3600
         pkg:
           - git
           - wget

ansible/cache.yml

@@ -0,0 +1,15 @@
+# SPDX-FileCopyrightText: 2023 Aravinth Manivannan <realaravinth@batsense.net>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+---
+- name: Base configuration
+  ansible.builtin.import_playbook: base.yml
+
+- name: Install redis cache
+  hosts: mcaptcha_hosts
+  remote_user: atm
+  pre_tasks:
+    - name: Ensure all VMs are reachable
+      ansible.builtin.ping:
+  roles:
+    - cache

ansible/roles/cache/handlers/main.yml

@@ -0,0 +1,7 @@
+- name: Restart redis
+  listen: restart redis
+  become: true
+  ansible.builtin.service:
+    name: redis
+    enabled: true
+    state: restarted

ansible/roles/cache/tasks/main.yml

@@ -0,0 +1,86 @@
+# SPDX-FileCopyrightText: 2023 Aravinth Manivannan <realaravinth@batsense.net>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+---
+- name: Install redis-server
+  become: true
+  ansible.builtin.apt:
+    update_cache: true
+    cache_valid_time: 3600
+    pkg:
+      - redis-server
+
+- name: Create Redis plugins dir
+  become: true
+  notify: restart redis
+  ansible.builtin.file:
+    path: /usr/lib/redis/modules
+    owner: redis
+    group: redis
+    state: directory
+    mode: "0755"
+
+- name: Load mCaptcha module
+  become: true
+  notify: restart redis
+  ansible.builtin.lineinfile:
+    dest: /etc/redis/redis.conf
+    line: "{{ item }}"
+  loop:
+    - "loadmodule /usr/lib/redis/modules/libcache.so"
+    - "bind 0.0.0.0"
+
+- name: Copy custom Redis configuration
+  become: true
+  notify: restart redis
+  ansible.builtin.file:
+    path: "{{ item }}"
+    owner: redis
+    group: redis
+    force: true
+  loop:
+    - /etc/redis/redis.conf
+    - /etc/redis/
+
+- name: Create download dir
+  ansible.builtin.file:
+    path: /tmp/cache-lib
+    state: directory
+    mode: "0755"
+
+- name: Download plugin
+  ansible.builtin.get_url:
+    url: https://dl.mcaptcha.org/mcaptcha/cache/master/cache-master-linux-amd64.tar.gz
+    dest: /tmp/cache-lib/
+    checksum: sha256:https://dl.mcaptcha.org/mcaptcha/cache/master/cache-master-linux-amd64.tar.gz.sha256
+
+- name: Extract cache-master-linux-amd64.tar.gz into /var/lib/foo
+  ansible.builtin.unarchive:
+    src: /tmp/cache-lib/cache-master-linux-amd64.tar.gz
+    remote_src: true
+    dest: /tmp/cache-lib/
+
+- name: Copy custom Redis configuration
+  become: true
+  notify: restart redis
+  ansible.builtin.copy:
+    src: /tmp/cache-lib/cache-master-linux-amd64/libcache.so
+    remote_src: true
+    dest: /usr/lib/redis/modules/
+    owner: redis
+    group: redis
+    force: true
+    mode: "0755"
+
+#    - name: Delete download dir
+#      ansible.builtin.file:
+#        path: /tmp/cache-lib
+#        state: absent
+
+- name: Allow port 6379 for redis
+  become: true
+  community.general.ufw:
+    state: enabled
+    rule: allow
+    proto: tcp
+    port: "6379"

ansible/roles/docker/tasks/main.yml

@@ -28,7 +28,7 @@
     name: docker-ce
     update_cache: true
 
-# - name: Install Docker Module for Python
+#- name: Install Docker Module for Python
 #  become: true
 #  community.general.pipx:
 #    name: docker

ansible/roles/docker_compose/tasks/main.yml

@@ -7,5 +7,6 @@
   become: true
   ansible.builtin.apt:
     update_cache: true
+    cache_valid_time: 3600
     pkg:
       - docker-compose

ansible/roles/ntp/tasks/main.yml

@@ -7,6 +7,7 @@
   become: true
   ansible.builtin.apt:
     update_cache: true
+    cache_valid_time: 3600
     pkg:
       - ntp
 

ansible/roles/ufw/tasks/main.yml

@@ -7,6 +7,7 @@
   become: true
   ansible.builtin.apt:
     update_cache: true
+    cache_valid_time: 3600
     pkg:
       - ufw
 

terraform/mcaptcha/.gitignore

@@ -0,0 +1,45 @@
+# Compiled files
+*.tfstate
+*.tfstate.backup
+*.tfstate.lock.info
+
+# logs
+*.log
+
+# Directories
+.terraform/
+.vagrant/
+
+# SSH Keys
+*.pem
+
+# Backup files
+*.bak
+
+# Ignored Terraform files
+*gitignore*.tf
+
+# Ignore Mac .DS_Store files
+.DS_Store
+
+# Ignored vscode files
+.vscode/
+
+# Ignore Any Generated JSON Files
+operations/automation-script/apply.json
+operations/automation-script/configversion.json
+operations/automation-script/run.template.json
+operations/automation-script/run.json
+operations/automation-script/variable.template.json
+operations/automation-script/variable.json
+operations/automation-script/workspace.template.json
+operations/automation-script/workspace.json
+operations/sentinel-policies-scripts/create-policy.template.json
+operations/sentinel-policies-scripts/create-policy.json
+operations/variable-scripts/variable.template.json
+operations/variable-scripts/variable.json
+
+# Sentinel runtime directory
+.sentinel
+dos
+hosts.ini

terraform/mcaptcha/.terraform.lock.hcl

@@ -0,0 +1,53 @@
+# This file is maintained automatically by "tofu init".
+# Manual edits may be lost in future updates.
+
+provider "registry.opentofu.org/dmacvicar/libvirt" {
+  version     = "0.7.6"
+  constraints = "~> 0.7.0"
+  hashes = [
+    "h1:mmbm4vTyC/DCGO4Ed/vbp5AKvy1gmVn/94fzB9VmR08=",
+    "zh:0bde54f6f658b20b620b875daf106b5b25b1bae4d15408d6c5f06d58360e254d",
+    "zh:0c97c6930015918b8a34b6d7a2b0c3d17a649c226fcd1874fcba5bbbc0f35972",
+    "zh:1bdd7aa0011c5f024a09a124836ee9bc8e71b05a6ece810c61824275fd3f695f",
+    "zh:2b0cc7c794e4caf395d84ffff0b380d17e4b3219a4696264271bfe5059450efe",
+    "zh:2f8633f7fe07f76c188836ed6f93321ec5fbf5c004bc7699e1741d9b21ed5f37",
+    "zh:5bf47eed286ce55ed10a5cf657de49a34ab21cc8677c56fef3aab69cdde41a27",
+    "zh:7dca790fc5fd1d42bc4bc7170be003a7093602026d0f95c8aab84ad551fdf2a4",
+    "zh:80476b68bc84e3d661d1390025f83879b88f9cdc836de9751af09bd5716089cb",
+    "zh:82f3e2f3f50176cd6041c8ba36e295cbda1b289ef52ab75b5eceb0f921f64f7b",
+    "zh:a179b165f3b9bb9a67ebbbf9d73157ded33f02d476b2f58906389dca03b653c9",
+    "zh:acae54a5d0616f22b3180ddd8e8aad39af664e604394fdacf1f7b337bca2d5b4",
+    "zh:da4406a2428a9a7e98272c032cb93431c3919253af2fe9934b532d26c0deab09",
+    "zh:f63dbd8e579ab5268d01ffab4503b8a8e736b70d1a04e4f271559ba8dd133dcd",
+    "zh:f85c1d9e51a94ecde137435c9d6b0fb7be590437ea8a725334d1577eebbc550c",
+  ]
+}
+
+provider "registry.opentofu.org/hashicorp/local" {
+  version = "2.4.0"
+  hashes = [
+    "h1:pWJMQ+uRtVtHg97vU2zSCuYcZTuDQ7FJz+QanfSGMXM=",
+    "zh:184d6ec1f0e77713b37f0d9cf943b1371f2aa2f44c2c5a618978e897ce3dccab",
+    "zh:2205a7955a4051c2c25e69646a60746d9416b73001491808ae5d10620f7b7ac1",
+    "zh:256ddc56457f725819dc6be62f2d0bb3b9fee40a61771317bb32353df5b5c1a0",
+    "zh:70146e603f540523f6fa2251dd52c225db5a92bda8c07fd198ed51ae2b50176b",
+    "zh:8c3f9fe12ab8843e25ff7edabc26e01df4a0e8db204e432600a4c77a95ec0535",
+    "zh:b003e421f643d14247d31dcb7f0f6470c46f772d0e15a175a555a525ce344bf2",
+    "zh:b4c8ad7c5696aeb2a52adf6047d1e01943fafa57dc123d5192542527406ffd72",
+    "zh:c3b6fbfa431f3c085621c74596ee63681a278fd433a4758f33c627e8936d5cb3",
+    "zh:d2e57b19295b326d84ca5f39b797849d901170d5509aa7558f2a6545c9ce72a9",
+    "zh:e2307421b0b380eb0e8fcee008e0af98ae30fccbfc9e9a1d24d952489e9b0df9",
+  ]
+}
+
+provider "registry.opentofu.org/hashicorp/template" {
+  version = "2.2.0"
+  hashes = [
+    "h1:tdS0otiAtvUV8uLJWJNfcqOPo3llj7FyRzExw6X1srY=",
+    "zh:374c28bafc43cd65e578cb209efc9eee4c1cec7618f451528e928db98059e8c8",
+    "zh:6a2982e70fbc2ab2668d624c648ef2eb32243c1a1185246b03991a7a21326db9",
+    "zh:af83169c21bb13f141510a349e1f70cf7d893247a269bd71cad74dd22f1df0f5",
+    "zh:b81a5bedc91a1a81b938c393247248d6c3d1bd8ea685541f9c858908c0afb6b3",
+    "zh:de15486244af2d29d44d510d647cd6e0b1408e89952261013c572b7c9bfd744b",
+  ]
+}

terraform/mcaptcha/cloud_init.cfg

@@ -0,0 +1,23 @@
+#cloud-config
+# vim: syntax=yaml
+
+users:
+- name: root
+  ssh_authorized_keys:
+    - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC/wXdHpwpY/4ubhYTmuNdGepQpj1kchvTUTApxMZyfyVW4uzrPRTYsle1y9QbTBV35qLkNajRC/wmC5/xPchdXpsJpuD9st1HMhLeR8qwaPyptiYJYT+z/WisWw2k6oWhG3QKvPoRtBdW9nhZnkG+O6zkuGXiRHpS7j2VVboDPpWEe1UdELQFVCwfraRal2g3ENFZ/9V1UrW/4ahRnQnSxERplZUm/fgSxQtmXubTkW68ut7yasBsrKFffMm8JztW0tWgTlTKONd3LCjv4juM0t5+cJDotNDnUR86Tq2PG8io7no/h8BWtazmjdpfGgn02ibX26BkdU0LDEYbJt5q9/Fh9TGk2ZwcMQeyepO1AWQgkmHXZWZELqu6MLQpqdtsOjHp9k0MeSpuIbdwzgf10Ydy7vK1z8irS24tVNNnJaMBwOlVOPwfyztHRADPkFcv2lKSjS1uyKR0FIkV8Kvs4txaIjmwv2LfMg6lF5W6j3ZPLyeE4cplJP0DDjzorSanu31xVnqVb3A8V9awsJ/4H7d59bI99c7QHL4K3fBVP3O0gqd31xAVRsdGs5Tj2P+RpiI6o5JJiOa1+DuBdWzrVIXYchQ30ZjaJp1wTNsYLmAsjeYuQZE2tf1xvywdzD4MB4avugDEWikzRWN9V5PHDZr1bamTCCjOrb2PRCd7eSQ== aravinth7820@gmail.com
+- name: atm
+  gecos: Aravinth Manivannan
+  groups: users, admin
+  sudo: ALL=(ALL) NOPASSWD:ALL
+  shell: /bin/bash
+  lock_passwd: true
+  plain_text_passwd: fooabr12
+  ssh_authorized_keys: 
+    - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC/wXdHpwpY/4ubhYTmuNdGepQpj1kchvTUTApxMZyfyVW4uzrPRTYsle1y9QbTBV35qLkNajRC/wmC5/xPchdXpsJpuD9st1HMhLeR8qwaPyptiYJYT+z/WisWw2k6oWhG3QKvPoRtBdW9nhZnkG+O6zkuGXiRHpS7j2VVboDPpWEe1UdELQFVCwfraRal2g3ENFZ/9V1UrW/4ahRnQnSxERplZUm/fgSxQtmXubTkW68ut7yasBsrKFffMm8JztW0tWgTlTKONd3LCjv4juM0t5+cJDotNDnUR86Tq2PG8io7no/h8BWtazmjdpfGgn02ibX26BkdU0LDEYbJt5q9/Fh9TGk2ZwcMQeyepO1AWQgkmHXZWZELqu6MLQpqdtsOjHp9k0MeSpuIbdwzgf10Ydy7vK1z8irS24tVNNnJaMBwOlVOPwfyztHRADPkFcv2lKSjS1uyKR0FIkV8Kvs4txaIjmwv2LfMg6lF5W6j3ZPLyeE4cplJP0DDjzorSanu31xVnqVb3A8V9awsJ/4H7d59bI99c7QHL4K3fBVP3O0gqd31xAVRsdGs5Tj2P+RpiI6o5JJiOa1+DuBdWzrVIXYchQ30ZjaJp1wTNsYLmAsjeYuQZE2tf1xvywdzD4MB4avugDEWikzRWN9V5PHDZr1bamTCCjOrb2PRCd7eSQ== aravinth7820@gmail.com
+
+ssh_pwauth: true
+chpasswd:
+  list: |
+     root:foobar12
+     atm:foobar12
+  expire: False

terraform/mcaptcha/main.tf

@@ -0,0 +1,46 @@
+# SPDX-FileCopyrightText: 2023 Aravinth Manivannan <realaravinth@batsense.net>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+terraform {
+  required_version = ">= 0.13"
+  required_providers {
+    libvirt = {
+      source  = "dmacvicar/libvirt"
+      version = "~> 0.7.0"
+    }
+  }
+}
+
+provider "libvirt" {
+  uri = var.libvirt_uri
+}
+
+resource "libvirt_pool" "mcaptcha_basic" {
+  name = "mcaptcha_basic"
+  type = "dir"
+  path = var.libvirt_pool_path
+
+}
+
+resource "libvirt_volume" "debian-mcaptcha-qcow2" {
+  name   = "debian-mcaptcha-qcow2"
+  pool   = libvirt_pool.mcaptcha_basic.name
+  source = var.libvirt_debian_src
+  format = "qcow2"
+}
+
+data "template_file" "user_data" {
+  template = file("${path.module}/cloud_init.cfg")
+}
+
+data "template_file" "network_config" {
+  template = file("${path.module}/network_config.cfg")
+}
+
+resource "libvirt_cloudinit_disk" "commoninit" {
+  name           = "commoninit.iso"
+  user_data      = data.template_file.user_data.rendered
+  network_config = data.template_file.network_config.rendered
+  pool           = libvirt_pool.mcaptcha_basic.name
+}

terraform/mcaptcha/mcaptcha.tf

@@ -0,0 +1,52 @@
+# SPDX-FileCopyrightText: 2023 Aravinth Manivannan <realaravinth@batsense.net>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+resource "libvirt_volume" "mcaptcha_volume" {
+  name           = "mcaptcha_volume-${count.index}"
+  base_volume_id = libvirt_volume.debian-mcaptcha-qcow2.id
+  count          = var.mcaptcha_vm_count
+  pool           = libvirt_pool.mcaptcha_basic.name
+  size           = var.mcaptcha_vm_disk_size
+}
+
+resource "libvirt_domain" "mcaptcha_mcaptcha" {
+  count = var.mcaptcha_vm_count
+
+  name   = "mcaptcha_mcaptcha_${count.index}"
+  memory = var.mcaptcha_vm_memory
+  vcpu   = var.mcaptcha_vm_vcpu
+
+  cloudinit = libvirt_cloudinit_disk.commoninit.id
+
+  console {
+    type        = "pty"
+    target_port = "0"
+    target_type = "serial"
+  }
+
+  console {
+    type        = "pty"
+    target_type = "virtio"
+    target_port = "1"
+  }
+
+  network_interface {
+    network_name   = "default"
+    wait_for_lease = true
+  }
+
+  disk {
+    volume_id = element(libvirt_volume.mcaptcha_volume.*.id, count.index)
+  }
+}
+
+locals {
+  mcaptcha_vm_ips   = [for i in libvirt_domain.mcaptcha_mcaptcha : i.network_interface.0.addresses[0]]
+  mcaptcha_vm_names = [for i in libvirt_domain.mcaptcha_mcaptcha : i.name]
+  mcaptcha_vm_map = [for i in libvirt_domain.mcaptcha_mcaptcha : {
+    ip   = i.network_interface.0.addresses[0],
+    name = i.name
+  }]
+}
+

terraform/mcaptcha/network_config.cfg

@@ -0,0 +1,4 @@
+version: 2
+ethernets:
+  ens3:
+    dhcp4: true

terraform/mcaptcha/output.tf

@@ -0,0 +1,20 @@
+# SPDX-FileCopyrightText: 2023 Aravinth Manivannan <realaravinth@batsense.net>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+
+output "mcaptcha_mcaptcha_ip" {
+  value = local.mcaptcha_vm_map
+}
+
+resource "local_file" "hosts_yml" {
+  content = templatefile("./templates/hosts.yml.tftpl",
+    {
+      mcaptcha_vms_ips   = local.mcaptcha_vm_ips,
+      mcaptcha_vms_names = local.mcaptcha_vm_names,
+      mcaptcha_vms      = local.mcaptcha_vm_map
+
+  })
+
+  filename = "hosts.ini"
+}

terraform/mcaptcha/scripts/on.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+
+for vm in $(virsh list --all --name --state-shutoff); do \
+	echo "[*] Starting vm: $vm"; \
+	virsh start $vm; \
+done

terraform/mcaptcha/templates/hosts.yml.tftpl

@@ -0,0 +1,4 @@
+[mcaptcha_hosts]
+%{ for vm in mcaptcha_vms ~}
+${vm.name} ansible_host=${vm.ip} ansible_user=atm
+%{ endfor ~}

terraform/mcaptcha/variables.tf

@@ -0,0 +1,46 @@
+# SPDX-FileCopyrightText: 2023 Aravinth Manivannan <realaravinth@batsense.net>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+/* main.tf */
+variable "libvirt_uri" {
+  description = "URI of libvert socket"
+  type        = string
+  default     = "qemu:///system"
+}
+
+variable "libvirt_pool_path" {
+  description = "Path of libvirt storage pool"
+  type        = string
+  default     = "/home/atm/code/libvirt/pool/mcaptcha_basic"
+}
+
+variable "libvirt_debian_src" {
+  description = "Location of Debian 11 qcow2 image"
+  type        = string
+  default     = "/home/atm/disk-images/debian/12/clould/debian-12-generic-amd64.qcow2"
+}
+
+/* mcaptcha/mcaptcha server  server */
+variable "mcaptcha_vm_count" {
+  description = "Number of VMs to be deployed to run mcaptcha/mcaptcha"
+  type        = number
+  default     = 1
+}
+
+variable "mcaptcha_vm_disk_size" {
+  description = "Size of disk of VMs running mcaptcha/mcaptcha in bytes"
+  type        = number
+  default     = 8000000000 # 8GB
+}
+
+variable "mcaptcha_vm_memory" {
+  description = "Memory of VMs running mcaptcha/mcaptcha in MB"
+  type        = number
+  default     = 2000 # 2GB
+}
+
+variable "mcaptcha_vm_vcpu" {
+  description = "Number of CPUs of VMs running mcaptcha/mcaptcha"
+  type        = number
+  default     = 4
+}

tests/cache/base.py

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: 2023 Aravinth Manivannan <realaravinth@batsense.net>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+import os
+
+
+def test_redis_is_listening(host):
+    socket = host.socket(f"tcp://0.0.0.0:6379")
+    assert socket.is_listening
+
+
+def test_redis_config_exists(host):
+    config = host.file("/etc/redis/redis.conf")
+    assert config.exists
+    assert config.is_file
+
+
+def test_redis_service_running_and_enabled(host):
+    service = host.service("redis")
+    assert service.is_running
+    assert service.is_enabled