Merge pull request 'Integration an unit tests' (#4) from feat-integration-tests into master

Reviewed-on: #4

.woodpecker.yml

@@ -5,10 +5,23 @@ steps:
       - apt update
       - apt-get install -y --no-install-recommends protobuf-compiler
       - cargo build
+      - cargo test --lib
       #      - make migrate
       #      - make
       #      - make release
         #      - make test // requires Docker-in-Docker
+  integration_tests:
+    image: python
+    commands:
+      - pip install virtualenv && virtualenv venv
+      - . venv/bin/activate && pip install -r requirements.txt
+      - nohup ./target/debug/main --id 1 --http-addr 127.0.0.1:9001 --introducer-addr 127.0.0.1:9001 --introducer-id 1  --cluster-size 3 &
+      - sleep 1
+      - nohup ./target/debug/main --id 2 --http-addr 127.0.0.1:9002 --introducer-addr 127.0.0.1:9001 --introducer-id 1  --cluster-size 3 &
+      - sleep 1
+      - nohup ./target/debug/main --id 3 --http-addr 127.0.0.1:9003 --introducer-addr 127.0.0.1:9001 --introducer-id 1  --cluster-size 3 &
+      - mv dcache_py/ tests/
+      - . venv/bin/activate && python tests/test.py
 
   build_docker_img:
     image: plugins/docker

requirements.txt

@@ -0,0 +1,32 @@
+asyncio==3.4.3
+blinker==1.7.0
+Brotli==1.1.0
+certifi==2023.11.17
+charset-normalizer==3.3.2
+click==8.1.7
+ConfigArgParse==1.7
+Flask==3.0.0
+Flask-BasicAuth==0.2.0
+Flask-Cors==4.0.0
+gevent==23.9.1
+geventhttpclient==2.0.11
+greenlet==3.0.2
+grpc-interceptor==0.15.4
+grpcio==1.60.0
+grpcio-tools==1.60.0
+idna==3.6
+itsdangerous==2.1.2
+Jinja2==3.1.2
+locust==2.20.0
+MarkupSafe==2.1.3
+msgpack==1.0.7
+protobuf==4.25.1
+psutil==5.9.7
+pyzmq==25.1.2
+requests==2.31.0
+roundrobin==0.0.4
+six==1.16.0
+urllib3==2.1.0
+Werkzeug==3.0.1
+zope.event==5.0
+zope.interface==6.1

tests/.gitignore

@@ -0,0 +1,138 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/

tests/bucket.py

@@ -0,0 +1,93 @@
+#!/bin/env /usr/bin/python3
+# # Copyright (C) 2021  Aravinth Manivannan <realaravinth@batsense.net>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from asyncio import sleep
+import sys
+import json
+
+from mcaptcha import register
+
+from dcache import grpc_add_vote, grpc_get_visitor_count
+
+
+def incr(key):
+    return grpc_add_vote(key)
+
+
+def get_count(key):
+    try:
+        count = grpc_get_visitor_count(key)
+        return int(count.visitors)
+    except:
+        return 0
+
+
+def assert_count(expect, key):
+    count = get_count(key)
+    assert count == expect
+
+
+async def incr_one_works():
+    try:
+        key = "incr_one"
+        register(key)
+        initial_count = get_count(key)
+        # incriment
+        incr(key)
+        assert_count(initial_count + 1, key)
+        # wait till expiry
+        await sleep(5 + 2)
+        assert_count(initial_count, key)
+        print("[*] Incr one works")
+    except Exception as e:
+        raise e
+
+
+async def race_works():
+    key = "race_works"
+    try:
+        register(key)
+        initial_count = get_count(key)
+        race_num = 200
+        for _ in range(race_num):
+            incr(key)
+        assert_count(initial_count + race_num, key)
+        # wait till expiry
+        await sleep(5 + 2)
+        assert_count(initial_count, key)
+        print("[*] Race works")
+    except Exception as e:
+        raise e
+
+
+async def difficulty_works():
+    key = "difficulty_works"
+    try:
+        register(key)
+        data = incr(key)
+        assert data.difficulty_factor == 50
+
+        for _ in range(501):
+            incr(key)
+        data = incr(key)
+        assert data.difficulty_factor == 500
+
+        await sleep(5 + 2)
+        data = incr(key)
+        assert data.difficulty_factor == 50
+
+        print("[*] Difficulty factor works")
+    except Exception as e:
+        raise e

tests/dcache.py

@@ -0,0 +1,126 @@
+import requests
+import grpc
+import json
+
+from dcache_py import dcache_pb2 as dcache
+from dcache_py.dcache_pb2 import RaftRequest
+from dcache_py.dcache_pb2_grpc import DcacheServiceStub
+
+host = "localhost:9001"
+
+
+def grpc_add_vote(captcha_id: str):
+    with grpc.insecure_channel(host) as channel:
+        stub = DcacheServiceStub(channel)
+
+        msg = dcache.CaptchaID(id=captcha_id)
+        resp = stub.AddVisitor(msg)
+        return resp.result
+
+
+def grpc_add_captcha(captcha_id: str):
+    with grpc.insecure_channel(host) as channel:
+        stub = DcacheServiceStub(channel)
+
+        msg = dcache.AddCaptchaRequest(
+            id=captcha_id,
+            mcaptcha=dcache.MCaptcha(
+                duration=5,
+                defense=dcache.Defense(
+                    levels=[
+                        dcache.Level(visitor_threshold=50, difficulty_factor=50),
+                        dcache.Level(visitor_threshold=500, difficulty_factor=500),
+                    ]
+                ),
+            ),
+        )
+
+        resp = stub.AddCaptcha(msg)
+        return resp
+
+
+def grpc_captcha_exists(captcha_id: str):
+    with grpc.insecure_channel(host) as channel:
+        stub = DcacheServiceStub(channel)
+        msg = dcache.CaptchaID(id=captcha_id)
+        resp = stub.CaptchaExists(msg)
+        return resp.exists
+
+
+def grpc_rename_captcha(captcha_id: str, new_id: str):
+    with grpc.insecure_channel(host) as channel:
+        stub = DcacheServiceStub(channel)
+
+        msg = dcache.RenameCaptchaRequest(name=captcha_id, rename_to=new_id)
+        resp = stub.RenameCaptcha(msg)
+
+
+def grpc_delete_captcha(captcha_id: str):
+    with grpc.insecure_channel(host) as channel:
+        stub = DcacheServiceStub(channel)
+
+        msg = dcache.CaptchaID(id=captcha_id)
+        stub.RemoveCaptcha(msg)
+
+
+def grpc_get_visitor_count(captcha_id: str):
+    with grpc.insecure_channel(host) as channel:
+        stub = DcacheServiceStub(channel)
+        msg = dcache.CaptchaID(id=captcha_id)
+        return stub.GetVisitorCount(msg).result
+
+
+def grpc_add_challenge(token: str, key: str):
+    with grpc.insecure_channel(host) as channel:
+        stub = DcacheServiceStub(channel)
+        msg = dcache.CacheResultRequest(
+            token=token,
+            key=key,
+            duration=5,
+        )
+        stub.CacheResult(msg)
+
+
+def grpc_get_challenge(token: str, key: str):
+    with grpc.insecure_channel(host) as channel:
+        stub = DcacheServiceStub(channel)
+        msg = dcache.RetrievePowRequest(
+            token=token,
+            key=key,
+        )
+        return stub.VerifyCaptchaResult(msg)
+
+
+def grpc_delete_challenge(token: str):
+    with grpc.insecure_channel(host) as channel:
+        stub = DcacheServiceStub(channel)
+        msg = dcache.DeleteCaptchaResultRequest(
+            token=token,
+        )
+        stub.DeleteCaptchaResult(msg)
+
+
+def grpc_add_pow(token: str, string: str):
+    with grpc.insecure_channel(host) as channel:
+        stub = DcacheServiceStub(channel)
+        msg = dcache.CachePowRequest(
+            key=token, string=string, duration=5, difficulty_factor=500
+        )
+        return stub.CachePow(msg)
+
+
+def grpc_get_pow(token: str, string: str):
+    with grpc.insecure_channel(host) as channel:
+        stub = DcacheServiceStub(channel)
+        msg = dcache.RetrievePowRequest(token=string, key=token)
+        resp = stub.RetrievePow(msg)
+        return resp
+
+
+def grpc_delete_pow(string: str):
+    with grpc.insecure_channel(host) as channel:
+        stub = DcacheServiceStub(channel)
+        msg = dcache.DeletePowRequest(
+            string=string,
+        )
+        stub.DeletePow(msg)

tests/mcaptcha.py

@@ -0,0 +1,87 @@
+#!/bin/env /usr/bin/python3
+#
+# Copyright (C) 2021  Aravinth Manivannan <realaravinth@batsense.net>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import json
+
+from dcache import (
+    grpc_add_captcha,
+    grpc_add_vote,
+    grpc_captcha_exists,
+    grpc_rename_captcha,
+)
+from dcache import grpc_delete_captcha
+
+def delete_captcha(key):
+    grpc_delete_captcha(key)
+
+
+def add_captcha(key):
+    grpc_add_captcha(key)
+
+
+def rename_captcha(key, new_key):
+    grpc_rename_captcha(key, new_key)
+
+
+def captcha_exists(key):
+    return grpc_captcha_exists(captcha_id=key)
+
+
+def register(key):
+    if captcha_exists(key):
+        delete_captcha(key)
+    add_captcha(key)
+
+
+async def captcha_exists_works():
+    key = "captcha_delete_works"
+    if captcha_exists(key):
+        delete_captcha(key)
+    assert captcha_exists(key) is False
+    register(key)
+    assert captcha_exists(key) is True
+    print("[*] Captcha delete works")
+
+
+async def register_captcha_works():
+    key = "register_captcha_works"
+    register(key)
+    assert captcha_exists(key) is True
+    print("[*] Add captcha works")
+
+
+async def delete_captcha_works():
+    key = "delete_captcha_works"
+    register(key)
+    exists = captcha_exists(key)
+    assert exists is True
+    delete_captcha(key)
+    assert captcha_exists(key) is False
+    print("[*] Delete captcha works")
+
+
+async def rename_captcha_works():
+    key = "rename_captcha_works"
+    new_key = "new_key_rename_captcha_works"
+    register(key)
+    exists = captcha_exists(key)
+    assert exists is True
+    rename_captcha(key, new_key)
+    print(captcha_exists(key))
+    assert captcha_exists(key) is False
+    assert captcha_exists(new_key) is True
+    print("[*] Rename captcha works")

tests/pow.py

@@ -0,0 +1,119 @@
+#!/bin/env /usr/bin/python3
+#
+# Copyright (C) 2023  Aravinth Manivannan <realaravinth@batsense.net>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from asyncio import sleep
+import json
+
+from dcache import grpc_add_pow, grpc_get_pow, grpc_delete_pow
+
+# 1. Check duplicate pow
+# 2. Create pow
+# 3. Read non-existent pow
+# 4. Read pow
+# 5. Read expired pow
+
+
+def add_pow(captcha, pow):
+    """Add pow to"""
+    try:
+        res = grpc_add_pow(captcha, pow)
+        return res
+    except Exception as e:
+        return e
+
+
+def get_pow_from(captcha, pow):
+    """Add pow to"""
+    try:
+        res = grpc_get_pow(captcha, pow)
+        if res.HasField("result"):
+            return res.result
+        else:
+            return None
+    except Exception as e:
+        return e
+
+
+def delete_pow(captcha, pow):
+    """Add pow to"""
+    try:
+        grpc_delete_pow(pow)
+    except Exception as e:
+        return e
+
+
+async def add_pow_works():
+    """Test: Add pow"""
+    try:
+        key = "add_pow"
+        pow_name = "add_pow_pow"
+
+        add_pow(key, pow_name)
+        stored_pow = get_pow_from(key, pow_name)
+        assert stored_pow.difficulty_factor == 500
+        assert stored_pow.duration == 5
+        print("[*] Add pow works")
+
+    except Exception as e:
+        raise e
+
+
+async def pow_ttl_works():
+    """Test: pow TTL"""
+    try:
+        key = "ttl_pow"
+        pow_name = "ttl_pow_pow"
+
+        add_pow(key, pow_name)
+        await sleep(5 + 2)
+
+        error = get_pow_from(key, pow_name)
+        assert error is None
+
+        print("[*] pow TTL works")
+    except Exception as e:
+        raise e
+
+
+async def pow_doesnt_exist():
+    """Test: Non-existent pow"""
+    try:
+        pow_name = "nonexistent_pow"
+        key = "nonexistent_pow_key"
+
+        error = get_pow_from(key, pow_name)
+        assert error is None
+
+        print("[*] pow Doesn't Exist works")
+    except Exception as e:
+        raise e
+
+
+async def delete_pow_works():
+    """Test: Delete pows"""
+    try:
+        pow_name = "delete_pow"
+        key = "delete_pow_key"
+        # pow = get_pow(pow_name)
+
+        add_pow(key, pow_name)
+        delete_pow(key, pow_name)
+        error = get_pow_from(key, pow_name)
+        assert error is None
+
+        print("[*] Delete pow works")
+    except Exception as e:
+        raise e

tests/result.py

@@ -0,0 +1,119 @@
+#!/bin/env /usr/bin/python3
+#
+# Copyright (C) 2023  Aravinth Manivannan <realaravinth@batsense.net>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from asyncio import sleep
+import json
+
+from dcache import grpc_add_challenge, grpc_get_challenge, grpc_delete_challenge
+
+# 1. Check duplicate result
+# 2. Create result
+# 3. Read non-existent result
+# 4. Read result
+# 5. Read expired result
+
+
+COMMANDS = {
+    "ADD": "MCAPTCHA_CACHE.ADD_result",
+    "GET": "MCAPTCHA_CACHE.GET_result",
+    "DEL": "MCAPTCHA_CACHE.DELETE_result",
+}
+
+result_NOT_FOUND = "result not found"
+DUPLICATE_result = "result already exists"
+REDIS_OK = bytes("OK", "utf-8")
+
+
+def add_result(captcha, result):
+    """Add result to"""
+    try:
+        grpc_add_challenge(captcha, result)
+    except Exception as e:
+        return e
+
+
+def get_result_from(captcha, result):
+    """Add result to"""
+    try:
+        return grpc_get_challenge(captcha, result)
+    except Exception as e:
+        return e
+
+
+def delete_result(captcha, result):
+    """Add result to"""
+    try:
+        grpc_delete_challenge(captcha)
+    except Exception as e:
+        return e
+
+
+async def add_result_works():
+    """Test: Add result"""
+    try:
+        key = "add_result"
+        result_name = "add_result_result"
+
+        add_result(key, result_name)
+        verified = get_result_from(key, result_name)
+        assert verified.verified is True
+        print("[*] Add result works")
+
+    except Exception as e:
+        raise e
+
+
+async def result_ttl_works():
+    """Test: result TTL"""
+    try:
+        key = "ttl_result"
+        result_name = "ttl_result_result"
+
+        add_result(key, result_name)
+        await sleep(5 + 2)
+
+        error = get_result_from(key, result_name)
+        #        assert str(error) == result_NOT_FOUND
+
+        print("[*] result TTL works")
+    except Exception as e:
+        raise e
+
+
+async def result_doesnt_exist():
+    """Test: Non-existent result"""
+    try:
+        result_name = "nonexistent_result"
+        key = "nonexistent_result_key"
+
+        error = get_result_from(key, result_name)
+        print("[*] result Doesn't Exist works")
+    except Exception as e:
+        raise e
+
+
+async def delete_result_works():
+    """Test: Delete results"""
+    try:
+        result_name = "delete_result"
+        key = "delete_result_key"
+
+        add_result(key, result_name)
+        resp = delete_result(key, result_name)
+
+        print("[*] Delete result works")
+    except Exception as e:
+        raise e

tests/runner.py

@@ -0,0 +1,65 @@
+#!/bin/env /usr/bin/python3
+# Copyright (C) 2023  Aravinth Manivannan <realaravinth@batsense.net>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from threading import Thread
+import asyncio
+
+import importlib.util
+import sys
+
+sys.path.append("/home/atm/code/mcaptcha/dcache/")
+
+import bucket
+import mcaptcha
+import result
+import pow
+
+
+class Runner(object):
+    __fn = [
+        bucket.incr_one_works,
+        bucket.race_works,
+        bucket.difficulty_works,
+        mcaptcha.delete_captcha_works,
+        mcaptcha.captcha_exists_works,
+        mcaptcha.register_captcha_works,
+        mcaptcha.rename_captcha_works,
+        result.add_result_works,
+        result.result_doesnt_exist,
+        result.result_ttl_works,
+        result.delete_result_works,
+        pow.add_pow_works,
+        pow.pow_doesnt_exist,
+        pow.pow_ttl_works,
+        pow.delete_pow_works,
+    ]
+    __tasks = []
+
+    async def __register(self):
+        """Register functions to be run"""
+        for fn in self.__fn:
+            task = asyncio.create_task(fn())
+            self.__tasks.append(task)
+
+    async def run(self):
+        """Wait for registered functions to finish executing"""
+        await self.__register()
+        for task in self.__tasks:
+            await task
+
+    """Runs in separate threads"""
+
+    def __init__(self):
+        super(Runner, self).__init__()

tests/test.py

@@ -0,0 +1,30 @@
+#!/bin/env /usr/bin/python3
+#
+# Copyright (C) 2021  Aravinth Manivannan <realaravinth@batsense.net>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import asyncio
+
+from runner import Runner
+
+
+async def main():
+    print("Running Integration Tests")
+    runner = Runner()
+    await runner.run()
+    print("All tests passed")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())