From dcc821782e740b2efee6054d9acc9fb823baaca3 Mon Sep 17 00:00:00 2001
From: Aravinth Manivannan <realaravinth@batsense.net>
Date: Sat, 9 Dec 2023 03:55:20 +0530
Subject: [PATCH 01/19] feat: init CI

---
 .woodpecker.yml | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 .woodpecker.yml

diff --git a/.woodpecker.yml b/.woodpecker.yml
new file mode 100644
index 0000000..347e353
--- /dev/null
+++ b/.woodpecker.yml
@@ -0,0 +1,13 @@
+steps:
+  backend:
+    image: python
+    commands:
+      - apt-get update
+      - apt-get install -y ca-certificates curl gnupg tar wget libssl-dev python3-pip cython3 pipx
+      - pipx install --include-deps ansible
+      - pipx ensurepath && echo $SHELL
+      - /root/.local/bin/ansible-playbook ./ansible/mcaptcha.yml --connection=local
+      - pipx install pytest-testinfra
+      - cd tests/cache/ && py.test --verbose base.py
+      - cd tests/mcaptcha/ && py.test --verbose base.py
+
-- 
2.39.5


From 69fb946f3f31710d0717dfefd5b13aa0bd9b34d9 Mon Sep 17 00:00:00 2001
From: Aravinth Manivannan <realaravinth@batsense.net>
Date: Sat, 9 Dec 2023 04:11:44 +0530
Subject: [PATCH 02/19] hotfix: run mcaptcha playbook on all hosts

---
 ansible/cache.yml    | 2 +-
 ansible/mcaptcha.yml | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/ansible/cache.yml b/ansible/cache.yml
index b5fff3d..6dfa3ab 100644
--- a/ansible/cache.yml
+++ b/ansible/cache.yml
@@ -6,7 +6,7 @@
   ansible.builtin.import_playbook: base.yml
 
 - name: Install redis cache
-  hosts: mcaptcha_hosts
+  hosts: all
   pre_tasks:
     - name: Ensure all VMs are reachable
       ansible.builtin.ping:
diff --git a/ansible/mcaptcha.yml b/ansible/mcaptcha.yml
index 45927d8..f18a502 100644
--- a/ansible/mcaptcha.yml
+++ b/ansible/mcaptcha.yml
@@ -6,7 +6,7 @@
   ansible.builtin.import_playbook: base.yml
 
 - name: Install and configure postgres
-  hosts: mcaptcha_hosts
+  hosts: all
   become: yes
   vars_files:
     - vars/mcaptcha/vars.yml
@@ -18,7 +18,7 @@
         database_type == "postgres"
 
 - name: Install and configure mariadb
-  hosts: mcaptcha_hosts
+  hosts: all
   become: yes
   vars_files:
     - vars/mcaptcha/vars.yml
@@ -30,7 +30,7 @@
         database_type == "mariadb"
 
 - name: Install and configure cache
-  hosts: mcaptcha_hosts
+  hosts: all
   become: yes
   vars_files:
     - vars/mcaptcha/vars.yml
@@ -41,7 +41,7 @@
       when: cache_type == "redis"
 
 - name: Install mCaptcha binary
-  hosts: mcaptcha_hosts
+  hosts: all
   vars_files:
     - vars/mcaptcha/vars.yml
   roles:
-- 
2.39.5


From b74731476d99ee86c89942ce68be1f9a24c184e9 Mon Sep 17 00:00:00 2001
From: Aravinth Manivannan <realaravinth@batsense.net>
Date: Sat, 9 Dec 2023 04:12:23 +0530
Subject: [PATCH 03/19] fix: testinfra install command

---
 .woodpecker.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.woodpecker.yml b/.woodpecker.yml
index 347e353..785ded3 100644
--- a/.woodpecker.yml
+++ b/.woodpecker.yml
@@ -7,7 +7,7 @@ steps:
       - pipx install --include-deps ansible
       - pipx ensurepath && echo $SHELL
       - /root/.local/bin/ansible-playbook ./ansible/mcaptcha.yml --connection=local
-      - pipx install pytest-testinfra
+      - pipx install --include-deps pytest-testinfra
       - cd tests/cache/ && py.test --verbose base.py
       - cd tests/mcaptcha/ && py.test --verbose base.py
 
-- 
2.39.5


From 2c02ae4b9546b810d8625f7203f4c53d8b116278 Mon Sep 17 00:00:00 2001
From: Aravinth Manivannan <realaravinth@batsense.net>
Date: Sat, 9 Dec 2023 04:25:38 +0530
Subject: [PATCH 04/19] fix: document roles

---
 .woodpecker.yml          | 4 ++--
 ansible/requirements.yml | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)
 create mode 100644 ansible/requirements.yml

diff --git a/.woodpecker.yml b/.woodpecker.yml
index 785ded3..2ecc025 100644
--- a/.woodpecker.yml
+++ b/.woodpecker.yml
@@ -6,8 +6,8 @@ steps:
       - apt-get install -y ca-certificates curl gnupg tar wget libssl-dev python3-pip cython3 pipx
       - pipx install --include-deps ansible
       - pipx ensurepath && echo $SHELL
-      - /root/.local/bin/ansible-playbook ./ansible/mcaptcha.yml --connection=local
+      - /root/.local/bin/ansible-galaxy install -r ./ansible/requirements.yml
+      - /root/.local/bin/ansible-playbook --connection=local -i localhost, --limit 127.0.0.1  ./ansible/mcaptcha.yml
       - pipx install --include-deps pytest-testinfra
       - cd tests/cache/ && py.test --verbose base.py
       - cd tests/mcaptcha/ && py.test --verbose base.py
-
diff --git a/ansible/requirements.yml b/ansible/requirements.yml
new file mode 100644
index 0000000..8629619
--- /dev/null
+++ b/ansible/requirements.yml
@@ -0,0 +1,3 @@
+- src: geerlingguy.mysql
+
+- src: geerlingguy.postgresql
-- 
2.39.5


From 66f39799301284fba625d41ee8eb0f52be9c669a Mon Sep 17 00:00:00 2001
From: Aravinth Manivannan <realaravinth@batsense.net>
Date: Sat, 9 Dec 2023 04:42:25 +0530
Subject: [PATCH 05/19] debug: become root

---
 .woodpecker.yml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/.woodpecker.yml b/.woodpecker.yml
index 2ecc025..e227495 100644
--- a/.woodpecker.yml
+++ b/.woodpecker.yml
@@ -1,13 +1,17 @@
 steps:
   backend:
-    image: python
+    image: python:3-bookworm
     commands:
+      - whoami
       - apt-get update
-      - apt-get install -y ca-certificates curl gnupg tar wget libssl-dev python3-pip cython3 pipx
+      - apt upgrade -y
+      - apt-get install -y ca-certificates curl gnupg tar wget libssl-dev python3-pip cython3 pipx ufw iptables
+      - update-alternatives --set iptables /usr/sbin/iptables-legacy
+      - ufw status
       - pipx install --include-deps ansible
       - pipx ensurepath && echo $SHELL
       - /root/.local/bin/ansible-galaxy install -r ./ansible/requirements.yml
-      - /root/.local/bin/ansible-playbook --connection=local -i localhost, --limit 127.0.0.1  ./ansible/mcaptcha.yml
+      - /root/.local/bin/ansible-playbook --become --connection=local -i localhost, --limit 127.0.0.1  ./ansible/mcaptcha.yml
       - pipx install --include-deps pytest-testinfra
       - cd tests/cache/ && py.test --verbose base.py
       - cd tests/mcaptcha/ && py.test --verbose base.py
-- 
2.39.5


From b6b58b61566500c0965daac2e67cd3634d64bc3d Mon Sep 17 00:00:00 2001
From: Aravinth Manivannan <realaravinth@batsense.net>
Date: Mon, 11 Dec 2023 14:38:03 +0530
Subject: [PATCH 06/19] fix: rm ufw from ci

---
 .woodpecker.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.woodpecker.yml b/.woodpecker.yml
index e227495..f1659a8 100644
--- a/.woodpecker.yml
+++ b/.woodpecker.yml
@@ -5,9 +5,7 @@ steps:
       - whoami
       - apt-get update
       - apt upgrade -y
-      - apt-get install -y ca-certificates curl gnupg tar wget libssl-dev python3-pip cython3 pipx ufw iptables
-      - update-alternatives --set iptables /usr/sbin/iptables-legacy
-      - ufw status
+      - apt-get install -y ca-certificates curl gnupg tar wget libssl-dev python3-pip cython3 pipx
       - pipx install --include-deps ansible
       - pipx ensurepath && echo $SHELL
       - /root/.local/bin/ansible-galaxy install -r ./ansible/requirements.yml
-- 
2.39.5


From 64302fed4c5cae72d0be257703a969b14b353aa1 Mon Sep 17 00:00:00 2001
From: Aravinth Manivannan <realaravinth@batsense.net>
Date: Mon, 11 Dec 2023 14:58:21 +0530
Subject: [PATCH 07/19] feat: install opentofu and check network access

---
 .woodpecker.yml | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/.woodpecker.yml b/.woodpecker.yml
index f1659a8..23fa2b9 100644
--- a/.woodpecker.yml
+++ b/.woodpecker.yml
@@ -2,10 +2,17 @@ steps:
   backend:
     image: python:3-bookworm
     commands:
-      - whoami
       - apt-get update
-      - apt upgrade -y
-      - apt-get install -y ca-certificates curl gnupg tar wget libssl-dev python3-pip cython3 pipx
+      - apt-get install -y ca-certificates curl gnupg tar wget libssl-dev python3-pip cython3 pipx apt-transport-https coreutils iputils-ping
+      - install -m 0755 -d /etc/apt/keyrings
+      - curl -fsSL https://packages.opentofu.org/opentofu/tofu/gpgkey | gpg --no-tty --batch --dearmor -o /etc/apt/keyrings/opentofu.gpg
+      - chmod a+r /etc/apt/keyrings/opentofu.gpg
+      - echo "deb [signed-by=/etc/apt/keyrings/opentofu.gpg] https://packages.opentofu.org/opentofu/tofu/any/ any main" > /etc/apt/sources.list.d/opentofu.list 
+        echo "deb-src [signed-by=/etc/apt/keyrings/opentofu.gpg] https://packages.opentofu.org/opentofu/tofu/any/ any main" >> /etc/apt/sources.list.d/opentofu.list 
+      - apt-get update
+      - apt-get install -y tofu
+      - ping -c 10 192.168.0.102
+      - ping -c 10 192.168.0.127
       - pipx install --include-deps ansible
       - pipx ensurepath && echo $SHELL
       - /root/.local/bin/ansible-galaxy install -r ./ansible/requirements.yml
-- 
2.39.5


From 3da25ea1c3709696d2bb6caee426a59de550495e Mon Sep 17 00:00:00 2001
From: Aravinth Manivannan <realaravinth@batsense.net>
Date: Mon, 11 Dec 2023 17:38:36 +0530
Subject: [PATCH 08/19] feat: provision test domain in hellbat hypervisor

---
 .gitignore                   |  1 +
 .woodpecker.yml              | 22 +++++++++++++++++-----
 terraform/mcaptcha/ci.tfvars |  4 ++++
 3 files changed, 22 insertions(+), 5 deletions(-)
 create mode 100644 terraform/mcaptcha/ci.tfvars

diff --git a/.gitignore b/.gitignore
index 08d7f3e..542a744 100644
--- a/.gitignore
+++ b/.gitignore
@@ -161,3 +161,4 @@ cython_debug/
 #.idea/
 ansible/credentials/
 terraform/mcaptcha/mcaptcha
+sec/
diff --git a/.woodpecker.yml b/.woodpecker.yml
index 23fa2b9..d7fd24e 100644
--- a/.woodpecker.yml
+++ b/.woodpecker.yml
@@ -3,16 +3,24 @@ steps:
     image: python:3-bookworm
     commands:
       - apt-get update
-      - apt-get install -y ca-certificates curl gnupg tar wget libssl-dev python3-pip cython3 pipx apt-transport-https coreutils iputils-ping
+      - apt-get install -y ca-certificates curl gnupg tar wget libssl-dev python3-pip cython3 pipx apt-transport-https coreutils iputils-ping openssh-client libvirt-clients
+      - echo ${TEST_NODE_SSH_KEY} > /tmp/ssh-key && chmod 600 /tmp/ssh-key
+      - cat /tmp/ssh-key
+      -  eval "$(ssh-agent -s)"
+      - ssh-add /tmp/ssh-key
+      - ssh mcaptcha-ci@192.168.0.102 "echo f"
+      - virsh -c qemu+ssh://mcaptcha-ci@192.168.0.102/system?sshauth=privkey&no_verify=1 list
       - install -m 0755 -d /etc/apt/keyrings
       - curl -fsSL https://packages.opentofu.org/opentofu/tofu/gpgkey | gpg --no-tty --batch --dearmor -o /etc/apt/keyrings/opentofu.gpg
       - chmod a+r /etc/apt/keyrings/opentofu.gpg
-      - echo "deb [signed-by=/etc/apt/keyrings/opentofu.gpg] https://packages.opentofu.org/opentofu/tofu/any/ any main" > /etc/apt/sources.list.d/opentofu.list 
-        echo "deb-src [signed-by=/etc/apt/keyrings/opentofu.gpg] https://packages.opentofu.org/opentofu/tofu/any/ any main" >> /etc/apt/sources.list.d/opentofu.list 
+      - echo "deb [signed-by=/etc/apt/keyrings/opentofu.gpg] https://packages.opentofu.org/opentofu/tofu/any/ any main" > /etc/apt/sources.list.d/opentofu.list
+        echo "deb-src [signed-by=/etc/apt/keyrings/opentofu.gpg] https://packages.opentofu.org/opentofu/tofu/any/ any main" >> /etc/apt/sources.list.d/opentofu.list
       - apt-get update
       - apt-get install -y tofu
-      - ping -c 10 192.168.0.102
-      - ping -c 10 192.168.0.127
+      - tofu -chdir=terraform/mcaptcha/ init
+      - tofu -chdir=terraform/mcaptcha/ plan -var-file="ci.tfvars" -out="mcaptcha"
+      - tofu -chdir=terraform/mcaptcha/ apply mcaptcha -var-file="ci.tfvars"
+      - cd ../../
       - pipx install --include-deps ansible
       - pipx ensurepath && echo $SHELL
       - /root/.local/bin/ansible-galaxy install -r ./ansible/requirements.yml
@@ -20,3 +28,7 @@ steps:
       - pipx install --include-deps pytest-testinfra
       - cd tests/cache/ && py.test --verbose base.py
       - cd tests/mcaptcha/ && py.test --verbose base.py
+      - echo yes | tofu -chdir=terraform/mcaptcha/ destroy  \
+        -var-file="ci.tfvars"
+      - shred /tmp/ssh-key && rm /tmp/ssh-key
+    secrets: [TEST_NODE_SSH_KEY]
diff --git a/terraform/mcaptcha/ci.tfvars b/terraform/mcaptcha/ci.tfvars
new file mode 100644
index 0000000..3205632
--- /dev/null
+++ b/terraform/mcaptcha/ci.tfvars
@@ -0,0 +1,4 @@
+libvirt_uri = "qemu+ssh://mcaptcha-ci@192.168.0.102/system?keyfile=/tmp/ssh-key&sshauth=privkey&no_verify=1"
+#known_hosts_verify=ignore&keyfile=/tmp/ssh-key&sshauth=privkey&no_verify=1"
+libvirt_pool_path = "/srv/libvirt/pool/mcaptcha_basic/"
+libvirt_debian_src = "https://images.ci.mcaptcha.org/debian-12-generic-amd64.qcow2"
-- 
2.39.5


From 418c792e83a6eaf5df6c60bc8ebdb7b043c90da1 Mon Sep 17 00:00:00 2001
From: Aravinth Manivannan <realaravinth@batsense.net>
Date: Mon, 11 Dec 2023 19:30:28 +0530
Subject: [PATCH 09/19] feat: load SSH key

---
 .woodpecker.yml              | 17 ++++----
 Makefile                     |  6 +++
 scripts/ci.sh                | 84 ++++++++++++++++++++++++++++++++++++
 terraform/mcaptcha/ci.tfvars |  2 +-
 4 files changed, 99 insertions(+), 10 deletions(-)
 create mode 100755 scripts/ci.sh

diff --git a/.woodpecker.yml b/.woodpecker.yml
index d7fd24e..9fd0ae8 100644
--- a/.woodpecker.yml
+++ b/.woodpecker.yml
@@ -3,13 +3,12 @@ steps:
     image: python:3-bookworm
     commands:
       - apt-get update
-      - apt-get install -y ca-certificates curl gnupg tar wget libssl-dev python3-pip cython3 pipx apt-transport-https coreutils iputils-ping openssh-client libvirt-clients
-      - echo ${TEST_NODE_SSH_KEY} > /tmp/ssh-key && chmod 600 /tmp/ssh-key
-      - cat /tmp/ssh-key
+      - apt-get install -y ca-certificates curl gnupg tar wget libssl-dev python3-pip cython3 pipx apt-transport-https coreutils iputils-ping openssh-client libvirt-clients genisoimage
+      - make ci.init
+      - cat /tmp/ci-ssh-id
       -  eval "$(ssh-agent -s)"
-      - ssh-add /tmp/ssh-key
-      - ssh mcaptcha-ci@192.168.0.102 "echo f"
-      - virsh -c qemu+ssh://mcaptcha-ci@192.168.0.102/system?sshauth=privkey&no_verify=1 list
+      - ssh-add /tmp/ci-ssh-id
+      - ssh -o StrictHostKeyChecking=accept-new mcaptcha-ci@192.168.0.102 "echo f"
       - install -m 0755 -d /etc/apt/keyrings
       - curl -fsSL https://packages.opentofu.org/opentofu/tofu/gpgkey | gpg --no-tty --batch --dearmor -o /etc/apt/keyrings/opentofu.gpg
       - chmod a+r /etc/apt/keyrings/opentofu.gpg
@@ -19,7 +18,7 @@ steps:
       - apt-get install -y tofu
       - tofu -chdir=terraform/mcaptcha/ init
       - tofu -chdir=terraform/mcaptcha/ plan -var-file="ci.tfvars" -out="mcaptcha"
-      - tofu -chdir=terraform/mcaptcha/ apply mcaptcha -var-file="ci.tfvars"
+      - tofu -chdir=terraform/mcaptcha/ apply "mcaptcha"
       - cd ../../
       - pipx install --include-deps ansible
       - pipx ensurepath && echo $SHELL
@@ -28,7 +27,7 @@ steps:
       - pipx install --include-deps pytest-testinfra
       - cd tests/cache/ && py.test --verbose base.py
       - cd tests/mcaptcha/ && py.test --verbose base.py
-      - echo yes | tofu -chdir=terraform/mcaptcha/ destroy  \
+      - echo yes | tofu  destroy  \
         -var-file="ci.tfvars"
-      - shred /tmp/ssh-key && rm /tmp/ssh-key
+      - make ci.clean
     secrets: [TEST_NODE_SSH_KEY]
diff --git a/Makefile b/Makefile
index 601d16e..a4f7aee 100644
--- a/Makefile
+++ b/Makefile
@@ -134,5 +134,11 @@ test.cache: ## Test cache configuration
 test.mcaptcha: ## Test mcaptcha configuration
 	$(call test_mcaptcha)
 
+ci.init:
+	./scripts/ci.sh --init "$$TEST_NODE_SSH_KEY"
+
+ci.clean:
+	./scripts/ci.sh --clean
+
 help: ## Prints help for targets with comments
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-].+:.*?## .*$$' | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
diff --git a/scripts/ci.sh b/scripts/ci.sh
new file mode 100755
index 0000000..c062c73
--- /dev/null
+++ b/scripts/ci.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+# ci.sh: Helper script to automate deployment operations on CI/CD
+# Copyright © 2022 Aravinth Manivannan <realaravinth@batsense.net>
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -xEeuo  pipefail
+#source $(pwd)/scripts/lib.sh
+
+readonly SSH_ID_FILE=/tmp/ci-ssh-id
+
+match_arg() {
+    if [ $1 == $2 ] || [ $1 == $3 ]
+    then
+        return 0
+    else
+        return 1
+    fi
+}
+
+help() {
+	cat << EOF
+USAGE: ci.sh [SUBCOMMAND]
+Helper script to automate deployment operations on CI/CD
+
+Subcommands
+
+  -c  --clean	cleanup secrets, SSH key and other runtime data
+  -i  --init <SSH_PRIVATE_KEY>	initialize environment, write SSH private to file
+  -h  --help	print this help menu
+EOF
+}
+
+# $1: SSH private key
+write_ssh(){
+	truncate --size 0 $SSH_ID_FILE
+	echo "$1" > $SSH_ID_FILE
+	chmod 600 $SSH_ID_FILE
+}
+
+
+clean() {
+	if [ -f $SSH_ID_FILE ]
+	then
+		shred $SSH_ID_FILE
+		rm $SSH_ID_FILE
+	fi
+}
+
+if (( "$#" < 1 ))
+then
+	help
+	exit -1
+fi
+
+
+if match_arg $1 '-i' '--init'
+then
+	if (( "$#" < 2 ))
+	then
+		help
+		exit -1
+	fi
+	write_ssh "$2"
+elif match_arg $1 '-c' '--clean'
+then
+	clean
+elif match_arg $1 '-h' '--help'
+then
+	help
+else
+	help
+fi
diff --git a/terraform/mcaptcha/ci.tfvars b/terraform/mcaptcha/ci.tfvars
index 3205632..dab49ce 100644
--- a/terraform/mcaptcha/ci.tfvars
+++ b/terraform/mcaptcha/ci.tfvars
@@ -1,4 +1,4 @@
-libvirt_uri = "qemu+ssh://mcaptcha-ci@192.168.0.102/system?keyfile=/tmp/ssh-key&sshauth=privkey&no_verify=1"
+libvirt_uri = "qemu+ssh://mcaptcha-ci@192.168.0.102/system?keyfile=/tmp/ci-ssh-id&sshauth=privkey&no_verify=1"
 #known_hosts_verify=ignore&keyfile=/tmp/ssh-key&sshauth=privkey&no_verify=1"
 libvirt_pool_path = "/srv/libvirt/pool/mcaptcha_basic/"
 libvirt_debian_src = "https://images.ci.mcaptcha.org/debian-12-generic-amd64.qcow2"
-- 
2.39.5


From 58cb71dab3f54c48ee83d5c83d1bf456bc81dafd Mon Sep 17 00:00:00 2001
From: Aravinth Manivannan <realaravinth@batsense.net>
Date: Mon, 11 Dec 2023 20:15:37 +0530
Subject: [PATCH 10/19] feat: CI: use local image mirror

---
 terraform/mcaptcha/ci.tfvars | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/mcaptcha/ci.tfvars b/terraform/mcaptcha/ci.tfvars
index dab49ce..258af47 100644
--- a/terraform/mcaptcha/ci.tfvars
+++ b/terraform/mcaptcha/ci.tfvars
@@ -1,4 +1,4 @@
 libvirt_uri = "qemu+ssh://mcaptcha-ci@192.168.0.102/system?keyfile=/tmp/ci-ssh-id&sshauth=privkey&no_verify=1"
 #known_hosts_verify=ignore&keyfile=/tmp/ssh-key&sshauth=privkey&no_verify=1"
 libvirt_pool_path = "/srv/libvirt/pool/mcaptcha_basic/"
-libvirt_debian_src = "https://images.ci.mcaptcha.org/debian-12-generic-amd64.qcow2"
+libvirt_debian_src = "http://192.168.0.102/debian-12-generic-amd64.qcow2"
-- 
2.39.5


From 9833f7fd986eae2ce281c08a69164e9f14123901 Mon Sep 17 00:00:00 2001
From: Aravinth Manivannan <realaravinth@batsense.net>
Date: Mon, 11 Dec 2023 20:43:33 +0530
Subject: [PATCH 11/19] debug: use pool dir in home

---
 terraform/mcaptcha/ci.tfvars | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/mcaptcha/ci.tfvars b/terraform/mcaptcha/ci.tfvars
index 258af47..06edb9f 100644
--- a/terraform/mcaptcha/ci.tfvars
+++ b/terraform/mcaptcha/ci.tfvars
@@ -1,4 +1,4 @@
 libvirt_uri = "qemu+ssh://mcaptcha-ci@192.168.0.102/system?keyfile=/tmp/ci-ssh-id&sshauth=privkey&no_verify=1"
 #known_hosts_verify=ignore&keyfile=/tmp/ssh-key&sshauth=privkey&no_verify=1"
-libvirt_pool_path = "/srv/libvirt/pool/mcaptcha_basic/"
+libvirt_pool_path = "/home/mcaptcha-ci/libvirt/pool/mcaptcha_basic"
 libvirt_debian_src = "http://192.168.0.102/debian-12-generic-amd64.qcow2"
-- 
2.39.5


From 0306798bad40793a0c1fc8af2ec96eef02634a58 Mon Sep 17 00:00:00 2001
From: Aravinth Manivannan <realaravinth@batsense.net>
Date: Tue, 12 Dec 2023 19:32:24 +0530
Subject: [PATCH 12/19] feat: generalize ssh key and install qemu agent for
 macvtap IP retrival

---
 terraform/mcaptcha/cloud_init.cfg | 11 +++++++++--
 terraform/mcaptcha/main.tf        |  3 +++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/terraform/mcaptcha/cloud_init.cfg b/terraform/mcaptcha/cloud_init.cfg
index 1a4f40b..688b077 100644
--- a/terraform/mcaptcha/cloud_init.cfg
+++ b/terraform/mcaptcha/cloud_init.cfg
@@ -1,10 +1,17 @@
 #cloud-config
 # vim: syntax=yaml
+packages:
+  - sudo
+  - qemu-guest-agent
+runcmd:
+  - [ systemctl, daemon-reload ]
+  - [ systemctl, enable, qemu-guest-agent.service ]
+  - [ systemctl, start, --wait, qemu-guest-agent.service ]
 
 users:
 - name: root
   ssh_authorized_keys:
-    - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC/wXdHpwpY/4ubhYTmuNdGepQpj1kchvTUTApxMZyfyVW4uzrPRTYsle1y9QbTBV35qLkNajRC/wmC5/xPchdXpsJpuD9st1HMhLeR8qwaPyptiYJYT+z/WisWw2k6oWhG3QKvPoRtBdW9nhZnkG+O6zkuGXiRHpS7j2VVboDPpWEe1UdELQFVCwfraRal2g3ENFZ/9V1UrW/4ahRnQnSxERplZUm/fgSxQtmXubTkW68ut7yasBsrKFffMm8JztW0tWgTlTKONd3LCjv4juM0t5+cJDotNDnUR86Tq2PG8io7no/h8BWtazmjdpfGgn02ibX26BkdU0LDEYbJt5q9/Fh9TGk2ZwcMQeyepO1AWQgkmHXZWZELqu6MLQpqdtsOjHp9k0MeSpuIbdwzgf10Ydy7vK1z8irS24tVNNnJaMBwOlVOPwfyztHRADPkFcv2lKSjS1uyKR0FIkV8Kvs4txaIjmwv2LfMg6lF5W6j3ZPLyeE4cplJP0DDjzorSanu31xVnqVb3A8V9awsJ/4H7d59bI99c7QHL4K3fBVP3O0gqd31xAVRsdGs5Tj2P+RpiI6o5JJiOa1+DuBdWzrVIXYchQ30ZjaJp1wTNsYLmAsjeYuQZE2tf1xvywdzD4MB4avugDEWikzRWN9V5PHDZr1bamTCCjOrb2PRCd7eSQ== aravinth7820@gmail.com
+    - ${ssh_public_key}
 - name: atm
   gecos: Aravinth Manivannan
   groups: users, admin
@@ -13,7 +20,7 @@ users:
   lock_passwd: true
   plain_text_passwd: fooabr12
   ssh_authorized_keys: 
-    - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC/wXdHpwpY/4ubhYTmuNdGepQpj1kchvTUTApxMZyfyVW4uzrPRTYsle1y9QbTBV35qLkNajRC/wmC5/xPchdXpsJpuD9st1HMhLeR8qwaPyptiYJYT+z/WisWw2k6oWhG3QKvPoRtBdW9nhZnkG+O6zkuGXiRHpS7j2VVboDPpWEe1UdELQFVCwfraRal2g3ENFZ/9V1UrW/4ahRnQnSxERplZUm/fgSxQtmXubTkW68ut7yasBsrKFffMm8JztW0tWgTlTKONd3LCjv4juM0t5+cJDotNDnUR86Tq2PG8io7no/h8BWtazmjdpfGgn02ibX26BkdU0LDEYbJt5q9/Fh9TGk2ZwcMQeyepO1AWQgkmHXZWZELqu6MLQpqdtsOjHp9k0MeSpuIbdwzgf10Ydy7vK1z8irS24tVNNnJaMBwOlVOPwfyztHRADPkFcv2lKSjS1uyKR0FIkV8Kvs4txaIjmwv2LfMg6lF5W6j3ZPLyeE4cplJP0DDjzorSanu31xVnqVb3A8V9awsJ/4H7d59bI99c7QHL4K3fBVP3O0gqd31xAVRsdGs5Tj2P+RpiI6o5JJiOa1+DuBdWzrVIXYchQ30ZjaJp1wTNsYLmAsjeYuQZE2tf1xvywdzD4MB4avugDEWikzRWN9V5PHDZr1bamTCCjOrb2PRCd7eSQ== aravinth7820@gmail.com
+    - ${ssh_public_key}
 
 ssh_pwauth: true
 chpasswd:
diff --git a/terraform/mcaptcha/main.tf b/terraform/mcaptcha/main.tf
index 8c7aa9d..9266b54 100644
--- a/terraform/mcaptcha/main.tf
+++ b/terraform/mcaptcha/main.tf
@@ -32,6 +32,9 @@ resource "libvirt_volume" "debian-mcaptcha-qcow2" {
 
 data "template_file" "user_data" {
   template = file("${path.module}/cloud_init.cfg")
+  vars = {
+    ssh_public_key = var.ssh_public_key
+  }
 }
 
 data "template_file" "network_config" {
-- 
2.39.5


From bf87080e33e9eb9f5d15c4797e33f88e03fb1d4e Mon Sep 17 00:00:00 2001
From: Aravinth Manivannan <realaravinth@batsense.net>
Date: Tue, 12 Dec 2023 19:34:05 +0530
Subject: [PATCH 13/19] feat: use macvtap to make VM accessible to CI machine

---
 terraform/mcaptcha/ci.tfvars          |  9 +++++----
 terraform/mcaptcha/mcaptcha.tf        | 12 ++++++++----
 terraform/mcaptcha/network_config.cfg |  1 +
 terraform/mcaptcha/output.tf          |  2 +-
 terraform/mcaptcha/variables.tf       | 12 ++++++++++++
 5 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/terraform/mcaptcha/ci.tfvars b/terraform/mcaptcha/ci.tfvars
index 06edb9f..f1617d3 100644
--- a/terraform/mcaptcha/ci.tfvars
+++ b/terraform/mcaptcha/ci.tfvars
@@ -1,4 +1,5 @@
-libvirt_uri = "qemu+ssh://mcaptcha-ci@192.168.0.102/system?keyfile=/tmp/ci-ssh-id&sshauth=privkey&no_verify=1"
-#known_hosts_verify=ignore&keyfile=/tmp/ssh-key&sshauth=privkey&no_verify=1"
-libvirt_pool_path = "/home/mcaptcha-ci/libvirt/pool/mcaptcha_basic"
-libvirt_debian_src = "http://192.168.0.102/debian-12-generic-amd64.qcow2"
+libvirt_uri                = "qemu+ssh://mcaptcha-ci@192.168.0.102/system?keyfile=/tmp/ci-ssh-id&sshauth=privkey&no_verify=1"
+libvirt_pool_path          = "/home/mcaptcha-ci/libvirt/pool/mcaptcha_basic"
+libvirt_debian_src         = "http://192.168.0.102/debian-12-generic-amd64.qcow2"
+macvtap_ethernet_interface = "enp2s0"
+ssh_public_key             = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBCYagT9/PXoeaUae7Z3BKOPKBiEvJsCTUIhPCcRp5fb mcaptcha-ci@hellbat.batsense.net"
diff --git a/terraform/mcaptcha/mcaptcha.tf b/terraform/mcaptcha/mcaptcha.tf
index 0c19bd5..26889c1 100644
--- a/terraform/mcaptcha/mcaptcha.tf
+++ b/terraform/mcaptcha/mcaptcha.tf
@@ -13,9 +13,10 @@ resource "libvirt_volume" "mcaptcha_volume" {
 resource "libvirt_domain" "mcaptcha_mcaptcha" {
   count = var.mcaptcha_vm_count
 
-  name   = "mcaptcha_mcaptcha_${count.index}"
-  memory = var.mcaptcha_vm_memory
-  vcpu   = var.mcaptcha_vm_vcpu
+  name       = "mcaptcha_mcaptcha_${count.index}"
+  memory     = var.mcaptcha_vm_memory
+  vcpu       = var.mcaptcha_vm_vcpu
+  qemu_agent = true
 
   cloudinit = libvirt_cloudinit_disk.commoninit.id
 
@@ -31,8 +32,9 @@ resource "libvirt_domain" "mcaptcha_mcaptcha" {
     target_port = "1"
   }
 
+
   network_interface {
-    network_name   = "default"
+    macvtap        = var.macvtap_ethernet_interface
     wait_for_lease = true
   }
 
@@ -41,6 +43,8 @@ resource "libvirt_domain" "mcaptcha_mcaptcha" {
   }
 }
 
+
+
 locals {
   mcaptcha_vm_ips   = [for i in libvirt_domain.mcaptcha_mcaptcha : i.network_interface.0.addresses[0]]
   mcaptcha_vm_names = [for i in libvirt_domain.mcaptcha_mcaptcha : i.name]
diff --git a/terraform/mcaptcha/network_config.cfg b/terraform/mcaptcha/network_config.cfg
index 5b2cbca..4223aaa 100644
--- a/terraform/mcaptcha/network_config.cfg
+++ b/terraform/mcaptcha/network_config.cfg
@@ -2,3 +2,4 @@ version: 2
 ethernets:
   ens3:
     dhcp4: true
+#	ip: 192.168.0.115
diff --git a/terraform/mcaptcha/output.tf b/terraform/mcaptcha/output.tf
index 688a67e..4093d9c 100644
--- a/terraform/mcaptcha/output.tf
+++ b/terraform/mcaptcha/output.tf
@@ -12,7 +12,7 @@ resource "local_file" "hosts_yml" {
     {
       mcaptcha_vms_ips   = local.mcaptcha_vm_ips,
       mcaptcha_vms_names = local.mcaptcha_vm_names,
-      mcaptcha_vms      = local.mcaptcha_vm_map
+      mcaptcha_vms       = local.mcaptcha_vm_map
 
   })
 
diff --git a/terraform/mcaptcha/variables.tf b/terraform/mcaptcha/variables.tf
index 6bbfab4..246f128 100644
--- a/terraform/mcaptcha/variables.tf
+++ b/terraform/mcaptcha/variables.tf
@@ -44,3 +44,15 @@ variable "mcaptcha_vm_vcpu" {
   type        = number
   default     = 4
 }
+
+variable "macvtap_ethernet_interface" {
+  description = "Ethernet interface on the host machine that can be used as macvtap"
+  type        = string
+  default     = "enp2s0"
+}
+
+variable "ssh_public_key" {
+  description = "Set up SSH login for this public key"
+  type        = string
+  default     = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC/wXdHpwpY/4ubhYTmuNdGepQpj1kchvTUTApxMZyfyVW4uzrPRTYsle1y9QbTBV35qLkNajRC/wmC5/xPchdXpsJpuD9st1HMhLeR8qwaPyptiYJYT+z/WisWw2k6oWhG3QKvPoRtBdW9nhZnkG+O6zkuGXiRHpS7j2VVboDPpWEe1UdELQFVCwfraRal2g3ENFZ/9V1UrW/4ahRnQnSxERplZUm/fgSxQtmXubTkW68ut7yasBsrKFffMm8JztW0tWgTlTKONd3LCjv4juM0t5+cJDotNDnUR86Tq2PG8io7no/h8BWtazmjdpfGgn02ibX26BkdU0LDEYbJt5q9/Fh9TGk2ZwcMQeyepO1AWQgkmHXZWZELqu6MLQpqdtsOjHp9k0MeSpuIbdwzgf10Ydy7vK1z8irS24tVNNnJaMBwOlVOPwfyztHRADPkFcv2lKSjS1uyKR0FIkV8Kvs4txaIjmwv2LfMg6lF5W6j3ZPLyeE4cplJP0DDjzorSanu31xVnqVb3A8V9awsJ/4H7d59bI99c7QHL4K3fBVP3O0gqd31xAVRsdGs5Tj2P+RpiI6o5JJiOa1+DuBdWzrVIXYchQ30ZjaJp1wTNsYLmAsjeYuQZE2tf1xvywdzD4MB4avugDEWikzRWN9V5PHDZr1bamTCCjOrb2PRCd7eSQ== aravinth7820@gmail.com"
+}
-- 
2.39.5


From 713ec1dd18ab0a491d1adf20172e90a3c02a2bb1 Mon Sep 17 00:00:00 2001
From: Aravinth Manivannan <realaravinth@batsense.net>
Date: Tue, 12 Dec 2023 19:45:29 +0530
Subject: [PATCH 14/19] feat: cleanup ci config and use tofu-generated hostfile
 in ansible

---
 .woodpecker.yml | 35 ++++++++++++-----------------------
 Makefile        | 12 ++++++++++++
 2 files changed, 24 insertions(+), 23 deletions(-)

diff --git a/.woodpecker.yml b/.woodpecker.yml
index 9fd0ae8..7cf44b6 100644
--- a/.woodpecker.yml
+++ b/.woodpecker.yml
@@ -1,33 +1,22 @@
 steps:
   backend:
     image: python:3-bookworm
+    environment:
+      - ANSIBLE_HOST_KEY_CHECKING=False
     commands:
-      - apt-get update
-      - apt-get install -y ca-certificates curl gnupg tar wget libssl-dev python3-pip cython3 pipx apt-transport-https coreutils iputils-ping openssh-client libvirt-clients genisoimage
+      - export PATH=$PATH:/root/.local/bin
+      - eval "$(ssh-agent -s)"
       - make ci.init
-      - cat /tmp/ci-ssh-id
-      -  eval "$(ssh-agent -s)"
-      - ssh-add /tmp/ci-ssh-id
-      - ssh -o StrictHostKeyChecking=accept-new mcaptcha-ci@192.168.0.102 "echo f"
-      - install -m 0755 -d /etc/apt/keyrings
-      - curl -fsSL https://packages.opentofu.org/opentofu/tofu/gpgkey | gpg --no-tty --batch --dearmor -o /etc/apt/keyrings/opentofu.gpg
-      - chmod a+r /etc/apt/keyrings/opentofu.gpg
-      - echo "deb [signed-by=/etc/apt/keyrings/opentofu.gpg] https://packages.opentofu.org/opentofu/tofu/any/ any main" > /etc/apt/sources.list.d/opentofu.list
-        echo "deb-src [signed-by=/etc/apt/keyrings/opentofu.gpg] https://packages.opentofu.org/opentofu/tofu/any/ any main" >> /etc/apt/sources.list.d/opentofu.list
-      - apt-get update
-      - apt-get install -y tofu
       - tofu -chdir=terraform/mcaptcha/ init
       - tofu -chdir=terraform/mcaptcha/ plan -var-file="ci.tfvars" -out="mcaptcha"
       - tofu -chdir=terraform/mcaptcha/ apply "mcaptcha"
-      - cd ../../
-      - pipx install --include-deps ansible
-      - pipx ensurepath && echo $SHELL
-      - /root/.local/bin/ansible-galaxy install -r ./ansible/requirements.yml
-      - /root/.local/bin/ansible-playbook --become --connection=local -i localhost, --limit 127.0.0.1  ./ansible/mcaptcha.yml
-      - pipx install --include-deps pytest-testinfra
-      - cd tests/cache/ && py.test --verbose base.py
-      - cd tests/mcaptcha/ && py.test --verbose base.py
-      - echo yes | tofu  destroy  \
-        -var-file="ci.tfvars"
+      - pip install virtualenv && virtualenv venv
+      - . venv/bin/activate && pip install ansible pytest-testinfra requests
+      - . venv/bin/activate && ansible-galaxy install -r ./ansible/requirements.yml
+      - . venv/bin/activate && ansible-playbook --become -i terraform/mcaptcha/hosts.ini ./ansible/mcaptcha.yml
+      - cat terraform/mcaptcha/hosts.ini
+      - export ANSIBLE_REMOTE_USER=root
+      - . venv/bin/activate && py.test --hosts="ansible://mcaptcha_hosts" --ansible-inventory=terraform/mcaptcha/hosts.ini --verbose tests/cache/base.py
+      - . venv/bin/activate && py.test --hosts="ansible://mcaptcha_hosts" --ansible-inventory=terraform/mcaptcha/hosts.ini --verbose tests/mcaptcha/base.py
       - make ci.clean
     secrets: [TEST_NODE_SSH_KEY]
diff --git a/Makefile b/Makefile
index a4f7aee..4539d40 100644
--- a/Makefile
+++ b/Makefile
@@ -135,9 +135,21 @@ test.mcaptcha: ## Test mcaptcha configuration
 	$(call test_mcaptcha)
 
 ci.init:
+	apt-get update
+	apt-get install -y ca-certificates curl gnupg tar wget libssl-dev python3-pip cython3 pipx apt-transport-https coreutils iputils-ping openssh-client libvirt-clients genisoimage
 	./scripts/ci.sh --init "$$TEST_NODE_SSH_KEY"
+	ssh-add /tmp/ci-ssh-id
+	ssh -o StrictHostKeyChecking=accept-new mcaptcha-ci@192.168.0.102 "echo f"
+	install -m 0755 -d /etc/apt/keyrings
+	curl -fsSL https://packages.opentofu.org/opentofu/tofu/gpgkey | gpg --no-tty --batch --dearmor -o /etc/apt/keyrings/opentofu.gpg
+	chmod a+r /etc/apt/keyrings/opentofu.gpg
+	echo "deb [signed-by=/etc/apt/keyrings/opentofu.gpg] https://packages.opentofu.org/opentofu/tofu/any/ any main" > /etc/apt/sources.list.d/opentofu.list
+	echo "deb-src [signed-by=/etc/apt/keyrings/opentofu.gpg] https://packages.opentofu.org/opentofu/tofu/any/ any main" >> /etc/apt/sources.list.d/opentofu.list
+	apt-get update
+	apt-get install -y tofu
 
 ci.clean:
+	echo yes | tofu  destroy  -var-file="ci.tfvars"
 	./scripts/ci.sh --clean
 
 help: ## Prints help for targets with comments
-- 
2.39.5


From 7e9f0a3633b08d99c00d0fce603938a7ce9b4643 Mon Sep 17 00:00:00 2001
From: Aravinth Manivannan <realaravinth@batsense.net>
Date: Tue, 12 Dec 2023 22:53:52 +0530
Subject: [PATCH 15/19] feat: test for support DB and cache combinations

---
 .woodpecker.yml | 21 +++++++++++----------
 Makefile        |  8 +++++++-
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/.woodpecker.yml b/.woodpecker.yml
index 7cf44b6..8289480 100644
--- a/.woodpecker.yml
+++ b/.woodpecker.yml
@@ -7,16 +7,17 @@ steps:
       - export PATH=$PATH:/root/.local/bin
       - eval "$(ssh-agent -s)"
       - make ci.init
-      - tofu -chdir=terraform/mcaptcha/ init
-      - tofu -chdir=terraform/mcaptcha/ plan -var-file="ci.tfvars" -out="mcaptcha"
-      - tofu -chdir=terraform/mcaptcha/ apply "mcaptcha"
-      - pip install virtualenv && virtualenv venv
-      - . venv/bin/activate && pip install ansible pytest-testinfra requests
-      - . venv/bin/activate && ansible-galaxy install -r ./ansible/requirements.yml
-      - . venv/bin/activate && ansible-playbook --become -i terraform/mcaptcha/hosts.ini ./ansible/mcaptcha.yml
       - cat terraform/mcaptcha/hosts.ini
-      - export ANSIBLE_REMOTE_USER=root
-      - . venv/bin/activate && py.test --hosts="ansible://mcaptcha_hosts" --ansible-inventory=terraform/mcaptcha/hosts.ini --verbose tests/cache/base.py
-      - . venv/bin/activate && py.test --hosts="ansible://mcaptcha_hosts" --ansible-inventory=terraform/mcaptcha/hosts.ini --verbose tests/mcaptcha/base.py
+      # db: mariadb cache: redis
+      - . venv/bin/activate && ansible-playbook --become -i terraform/mcaptcha/hosts.ini --extra-vars "database_type=mariadb cache_type=redis" ./ansible/mcaptcha.yml
+      - INVENTORY=terraform/mcaptcha/hosts.ini make test.cache
+      - INVENTORY=terraform/mcaptcha/hosts.ini make test.mcaptcha
+      # db: postgres cache: redis
+      - . venv/bin/activate && ansible-playbook --become -i terraform/mcaptcha/hosts.ini --extra-vars "database_type=postgres cache_type=redis" ./ansible/mcaptcha.yml
+      - INVENTORY=terraform/mcaptcha/hosts.ini make test.cache
+      - INVENTORY=terraform/mcaptcha/hosts.ini make test.mcaptcha
+      # embedded cache
+      - . venv/bin/activate && ansible-playbook --become -i terraform/mcaptcha/hosts.ini --extra-vars "database_type=postgres cache_type=embedded" ./ansible/mcaptcha.yml
+      - INVENTORY=terraform/mcaptcha/hosts.ini make test.mcaptcha
       - make ci.clean
     secrets: [TEST_NODE_SSH_KEY]
diff --git a/Makefile b/Makefile
index 4539d40..e8a0343 100644
--- a/Makefile
+++ b/Makefile
@@ -147,9 +147,15 @@ ci.init:
 	echo "deb-src [signed-by=/etc/apt/keyrings/opentofu.gpg] https://packages.opentofu.org/opentofu/tofu/any/ any main" >> /etc/apt/sources.list.d/opentofu.list
 	apt-get update
 	apt-get install -y tofu
+	tofu -chdir=terraform/mcaptcha/ init
+	tofu -chdir=terraform/mcaptcha/ plan -var-file="ci.tfvars" -out="mcaptcha"
+	tofu -chdir=terraform/mcaptcha/ apply "mcaptcha"
+	pip install virtualenv && virtualenv venv
+	. venv/bin/activate && pip install ansible pytest-testinfra requests
+	. venv/bin/activate && ansible-galaxy install -r ./ansible/requirements.yml
 
 ci.clean:
-	echo yes | tofu  destroy  -var-file="ci.tfvars"
+	echo yes | tofu -chdir=terraform/mcaptcha/  destroy  -var-file="ci.tfvars"
 	./scripts/ci.sh --clean
 
 help: ## Prints help for targets with comments
-- 
2.39.5


From 1c17a3f3d99b5f0a64c0a066c230b7181252771e Mon Sep 17 00:00:00 2001
From: Aravinth Manivannan <realaravinth@batsense.net>
Date: Tue, 12 Dec 2023 23:52:43 +0530
Subject: [PATCH 16/19] feat: make mcaptcha systemd service wait on deps

---
 ansible/roles/mcaptcha/tasks/main.yml         |  8 +++---
 .../mcaptcha/mcaptcha.service.j2}             | 25 +++++++++++--------
 2 files changed, 19 insertions(+), 14 deletions(-)
 rename ansible/{artifacts/mcaptcha/mcaptcha.service => templates/mcaptcha/mcaptcha.service.j2} (61%)

diff --git a/ansible/roles/mcaptcha/tasks/main.yml b/ansible/roles/mcaptcha/tasks/main.yml
index 4852cfa..6ee38b9 100644
--- a/ansible/roles/mcaptcha/tasks/main.yml
+++ b/ansible/roles/mcaptcha/tasks/main.yml
@@ -42,9 +42,9 @@
 
 - name: Copy mCaptcha systemd servicefile
   become: true
-  ansible.builtin.copy:
-    src: ./artifacts/mcaptcha/mcaptcha.service
-    dest: /etc/systemd/system/
+  ansible.builtin.template:
+    src: ./templates/mcaptcha/mcaptcha.service.j2
+    dest: /etc/systemd/system/mcaptcha.service
     owner: root
     group: root
     force: true
@@ -57,7 +57,7 @@
     state: directory
     mode: "0755"
 
-- name: Copy mCaptcha systemd servicefile
+- name: Copy mCaptcha config file
   become: true
   notify: restart mcaptcha
   ansible.builtin.template:
diff --git a/ansible/artifacts/mcaptcha/mcaptcha.service b/ansible/templates/mcaptcha/mcaptcha.service.j2
similarity index 61%
rename from ansible/artifacts/mcaptcha/mcaptcha.service
rename to ansible/templates/mcaptcha/mcaptcha.service.j2
index 9889943..49dac32 100644
--- a/ansible/artifacts/mcaptcha/mcaptcha.service
+++ b/ansible/templates/mcaptcha/mcaptcha.service.j2
@@ -2,18 +2,23 @@
 Description=mCaptcha: PoW CAPTCHA system
 After=syslog.target
 After=network.target
-###
-# Don't forget to add the database service dependencies
-###
-#
-#Wants=mariadb.service
-#After=mariadb.service
-#
+
+{% if cache_type == 'redis' %}
+Wants=redis.service
+After=redis.service
+{% endif %}
+
+{% if database_type == 'postgres' %}
 Wants=postgresql.service
 After=postgresql.service
-#
-#Wants=redis.service
-#After=redis.service
+{% endif %}
+
+{% if database_type == 'mariadb' %}
+Wants=mariadb.service
+After=mariadb.service
+{% endif %}
+
+
 
 [Service]
 RestartSec=2s
-- 
2.39.5


From 165b2d24f7bc7d1c72f88bdb6cab63f445d153df Mon Sep 17 00:00:00 2001
From: Aravinth Manivannan <realaravinth@batsense.net>
Date: Tue, 12 Dec 2023 23:52:58 +0530
Subject: [PATCH 17/19] fix: typo

---
 ansible/mcaptcha.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/mcaptcha.yml b/ansible/mcaptcha.yml
index f18a502..b8717e0 100644
--- a/ansible/mcaptcha.yml
+++ b/ansible/mcaptcha.yml
@@ -22,7 +22,7 @@
   become: yes
   vars_files:
     - vars/mcaptcha/vars.yml
-    - vars/mcaptcha/mariadb.yml.yml
+    - vars/mcaptcha/mariadb.yml
   tasks:
     - ansible.builtin.include_role:
         name: geerlingguy.mysql
-- 
2.39.5


From 06d3da8019e126467e18f6476b1add228409eef1 Mon Sep 17 00:00:00 2001
From: Aravinth Manivannan <realaravinth@batsense.net>
Date: Tue, 12 Dec 2023 23:53:25 +0530
Subject: [PATCH 18/19] feat: install python deps from requirements.txt

---
 Makefile         |   2 +-
 requirements.txt | 296 +++--------------------------------------------
 2 files changed, 20 insertions(+), 278 deletions(-)

diff --git a/Makefile b/Makefile
index e8a0343..31c26c1 100644
--- a/Makefile
+++ b/Makefile
@@ -151,7 +151,7 @@ ci.init:
 	tofu -chdir=terraform/mcaptcha/ plan -var-file="ci.tfvars" -out="mcaptcha"
 	tofu -chdir=terraform/mcaptcha/ apply "mcaptcha"
 	pip install virtualenv && virtualenv venv
-	. venv/bin/activate && pip install ansible pytest-testinfra requests
+	. venv/bin/activate && pip install -r requirements.txt
 	. venv/bin/activate && ansible-galaxy install -r ./ansible/requirements.yml
 
 ci.clean:
diff --git a/requirements.txt b/requirements.txt
index 9363b82..be63dae 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,289 +1,31 @@
-ansible==8.1.0
-ansible-core==2.15.1
-certifi==2023.5.7
-cffi==1.15.1
-charset-normalizer==3.2.0
+ansible==9.1.0
+ansible-core==2.16.2
+certifi==2023.11.17
+cffi==1.16.0
+charset-normalizer==3.3.2
 contextvars==2.4
-cryptography==41.0.2
+cryptography==41.0.7
 distro==1.8.0
 execnet==2.0.2
-idna==3.4
-immutables==0.19
+idna==3.6
+immutables==0.20
 iniconfig==2.0.0
 Jinja2==3.1.2
 jmespath==1.0.1
 looseversion==1.3.0
 MarkupSafe==2.1.3
-msgpack==1.0.5
-packaging==23.1
-pluggy==1.2.0
-psutil==5.9.5
+msgpack==1.0.7
+packaging==23.2
+pluggy==1.3.0
+psutil==5.9.6
 pycparser==2.21
-pycryptodomex==3.18.0
-pytest==7.4.0
-pytest-testinfra==8.1.0
-pytest-xdist==3.3.1
-PyYAML==6.0
-pyzmq==25.0.2
-requests==2.31.0
-resolvelib==1.0.1
-salt==3006.1
-urllib3==2.0.3
-airdrop-ng==1.1
-airgraph-ng==1.1
-apparmor==3.1.6
-appdirs==1.4.4
-argcomplete==2.0.0
-asttokens==2.2.1
-async-timeout==4.0.2
-attrs==22.2.0
-autocommand==2.2.2
-autopage==0.5.1
-Babel==2.12.1
-beautifulsoup4==4.12.2
-binwalk==2.3.3
-black==23.7.0
-borgbackup==1.2.4
-Brlapi==0.8.5
-btrfsutil==6.3.3
-certifi==2023.7.22
-cffi==1.15.1
-chardet==5.2.0
-charset-normalizer==3.2.0
-click==8.1.6
-cliff==4.3.0
-cmd2==2.4.3
-codespell==2.2.5
-colorama==0.4.6
-ConfigArgParse==1.5.3
-contourpy==1.1.0
-crit==3.18
-cryptography==41.0.3
-cupshelpers==1.0
-cycler==0.11.0
-Cython==3.0.0
-dbus-python==1.3.2
-debtcollector==2.5.0
-decorator==5.1.1
-defusedxml==0.7.1
-deprecation==2.1.0
-devtools==0.11.0
-distlib==0.3.7
-distro==1.8.0
-dnspython==2.3.0
-dogpile.cache==1.1.8
-executing==1.2.0
-fastjsonschema==2.18.0
-filelock==3.12.2
-fonttools==4.42.0
-gevent==22.10.2
-git-filter-repo==2.38.0
-gpg==1.21.0
-greenlet==2.0.2
-html5lib==1.1
-httpie==3.2.1
-idna==3.4
-importlib-metadata==5.0.0
-inflect==7.0.0
-iotop==0.6
-iso8601==2.0.0
-jaraco.context==4.3.0
-jaraco.functools==3.8.1
-jaraco.text==3.11.1
-jedi==0.18.2
-jmespath==1.0.1
-jsonpatch==1.33
-jsonpointer==2.4
-jsonschema==4.19.0
-jsonschema-specifications==2023.7.1
-keystoneauth1==5.2.1
-kiwisolver==1.4.4
-lensfun==0.3.4
-LibAppArmor==3.1.6
-libfdt==1.7.0
-libtorrent==2.0.9
-libvirt-python==9.6.0
-lit==15.0.7.dev0
-louis==3.26.0
-lxml==4.9.2
-Mako==1.2.4
-mallard-ducktype==1.0.2
-Markdown==3.4.4
-markdown-it-py==2.2.0
-MarkupSafe==2.1.3
-matplotlib==3.7.2
-mdurl==0.1.2
-MemoizeDB==2021.11.20.2.41.2
-meson==1.2.1
-more-itertools==10.1.0
-msgpack==1.0.5
-multidict==6.0.4
-mypy-extensions==1.0.0
-netaddr==0.8.0
-netifaces==0.11.0
-nftables==0.1
-numpy==1.25.1
-openshot-qt==3.1.1
-openstacksdk==1.0.1
-ordered-set==4.1.0
-os-service-types==1.7.0
-osc-lib==2.8.0
-oslo.config==9.1.1
-oslo.i18n==6.0.0
-oslo.serialization==5.1.1
-oslo.utils==6.1.0
-packaging==23.1
-parso==0.8.3
-pathspec==0.11.2
-pbr==5.11.1
-perf==0.1
-Pillow==10.0.0
-pipenv==2023.7.23
-pipx==1.2.0
-platformdirs==3.10.0
-ply==3.11
-pm2ml==2021.11.20.2.41.2
-pooch==1.7.0
-Powerpill==2021.11.20.2.41.2
-prettytable==3.6.0
-protobuf==4.23.4
-psutil==5.9.5
-pwquality==1.4.5
-pyalpm==0.10.6
-pyasn1==0.4.8
-pybind11==2.11.1
-pycairo==1.24.0
-pycparser==2.21
-pycups==2.0.1
-pycurl==7.45.2
-pydantic==1.10.9
-Pygments==2.16.1
-PyGObject==3.44.1
-pynvim==0.4.3
-pyOpenSSL==23.2.0
-pyparsing==3.0.9
-pyperclip==1.8.2
-PyQt5==5.15.9
-PyQt5-sip==12.12.2
-PyQtWebEngine==5.15.6
-pyrsistent==0.19.3
-pysequoia==0.1.20
-PySocks==1.7.1
-python-cinderclient==9.3.0
-python-dateutil==2.8.2
-python-glanceclient==4.1.0
-python-keystoneclient==5.1.0
-python-novaclient==18.3.0
-python-openstackclient==6.2.0
-pytz==2023.3
-pyxdg==0.28
+pycryptodomex==3.19.0
+pytest==7.4.3
+pytest-testinfra==10.0.0
+pytest-xdist==3.5.0
 PyYAML==6.0.1
-pyzmq==25.1.1
-redis==4.6.0
-referencing==0.30.2
-Reflector==2023.6.28.0.36.1
-requests==2.28.2
-requests-toolbelt==1.0.0
-requestsexceptions==1.4.0
-rfc3986==2.0.0
-rich==13.5.2
-rpds-py==0.9.2
-rpm==4.18.1
-scipy==1.11.1
-simplejson==3.19.1
-six==1.16.0
-soupsieve==2.4.1
-stevedore==5.1.0
-TBB==0.2
-tomli==2.0.1
-torbrowser-launcher==0.3.6
-tqdm==4.65.0
-trash-cli==0.23.2.13.2
-trimage==1.0.6
-trove-classifiers==2023.8.8
-typing_extensions==4.7.1
-uc-micro-py==1.0.2
-ufw==0.36.2
-urllib3==1.26.15
-userpath==1.8.0
-validate-pyproject==0.13.post1.dev0+gb752273.d20230520
-vboxapi==1.0
-virtualenv==20.24.3
-virtualenv-clone==0.5.7
-warlock==2.0.1
-wcwidth==0.2.6
-webencodings==0.5.1
-wrapt==1.14.1
-XCGF==2021.11.20.2.41.3
-XCPF==2021.12.24.10.22.41
-youtube-dl==2021.12.17
-yt-dlp==2023.7.6
-zipp==3.16.2
-zope.event==5.0
-zope.interface==6.0
-ansible==8.1.0
-ansible-compat==4.1.6
-ansible-core==2.15.1
-ansible-lint==6.17.2
-astroid==2.15.6
-attrs==23.1.0
-black==23.7.0
-bracex==2.3.post1
-certifi==2023.5.7
-cffi==1.15.1
-charset-normalizer==3.2.0
-click==8.1.6
-contextvars==2.4
-cryptography==41.0.2
-dill==0.3.7
-distro==1.8.0
-execnet==2.0.2
-filelock==3.12.2
-gitdb==4.0.10
-GitPython==3.1.32
-idna==3.4
-immutables==0.19
-iniconfig==2.0.0
-isort==5.12.0
-Jinja2==3.1.2
-jmespath==1.0.1
-jsonschema==4.19.0
-jsonschema-specifications==2023.7.1
-lazy-object-proxy==1.9.0
-lint==1.2.1
-looseversion==1.3.0
-markdown-it-py==3.0.0
-MarkupSafe==2.1.3
-mccabe==0.7.0
-mdurl==0.1.2
-msgpack==1.0.5
-mypy-extensions==1.0.0
-packaging==23.1
-pathspec==0.11.2
-platformdirs==3.10.0
-pluggy==1.2.0
-psutil==5.9.5
-pycparser==2.21
-pycryptodomex==3.18.0
-Pygments==2.16.1
-pylint==2.17.5
-pytest==7.4.0
-pytest-testinfra==8.1.0
-pytest-xdist==3.3.1
-PyYAML==6.0
-pyzmq==25.0.2
-referencing==0.30.2
+pyzmq==25.1.2
 requests==2.31.0
 resolvelib==1.0.1
-rich==13.5.2
-rpds-py==0.9.2
-ruamel.yaml==0.17.32
-ruamel.yaml.clib==0.2.7
-salt==3006.1
-smmap==5.0.0
-subprocess-tee==0.4.1
-tomlkit==0.12.1
-urllib3==2.0.3
-wcmatch==8.4.1
-wrapt==1.15.0
-yamllint==1.32.0
+salt==3006.4
+urllib3==2.1.0
-- 
2.39.5


From 152fb05cb650dab1986505d11ec16183e53301bf Mon Sep 17 00:00:00 2001
From: Aravinth Manivannan <realaravinth@batsense.net>
Date: Wed, 13 Dec 2023 00:17:49 +0530
Subject: [PATCH 19/19] feat: test mariadb+embedded combination

---
 .woodpecker.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.woodpecker.yml b/.woodpecker.yml
index 8289480..41d1ae5 100644
--- a/.woodpecker.yml
+++ b/.woodpecker.yml
@@ -19,5 +19,9 @@ steps:
       # embedded cache
       - . venv/bin/activate && ansible-playbook --become -i terraform/mcaptcha/hosts.ini --extra-vars "database_type=postgres cache_type=embedded" ./ansible/mcaptcha.yml
       - INVENTORY=terraform/mcaptcha/hosts.ini make test.mcaptcha
+      # embedded cache
+      - . venv/bin/activate && ansible-playbook --become -i terraform/mcaptcha/hosts.ini --extra-vars "database_type=mariadb cache_type=embedded" ./ansible/mcaptcha.yml
+      - INVENTORY=terraform/mcaptcha/hosts.ini make test.mcaptcha
+
       - make ci.clean
     secrets: [TEST_NODE_SSH_KEY]
-- 
2.39.5