From c72688656fccac26c7f46ec6d7ca5942f82c8a88 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Sun, 31 Dec 2023 01:19:53 +0530 Subject: [PATCH 1/9] feat: add integration tests --- tests/.gitignore | 138 ++++++++++++++++++++++++++++++++++++++++++++++ tests/bucket.py | 89 ++++++++++++++++++++++++++++++ tests/dcache.py | 128 ++++++++++++++++++++++++++++++++++++++++++ tests/mcaptcha.py | 80 +++++++++++++++++++++++++++ tests/pow.py | 115 ++++++++++++++++++++++++++++++++++++++ tests/result.py | 115 ++++++++++++++++++++++++++++++++++++++ tests/runner.py | 63 +++++++++++++++++++++ tests/test.py | 29 ++++++++++ 8 files changed, 757 insertions(+) create mode 100644 tests/.gitignore create mode 100644 tests/bucket.py create mode 100644 tests/dcache.py create mode 100644 tests/mcaptcha.py create mode 100644 tests/pow.py create mode 100644 tests/result.py create mode 100644 tests/runner.py create mode 100755 tests/test.py diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..a81c8ee --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1,138 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ diff --git a/tests/bucket.py b/tests/bucket.py new file mode 100644 index 0000000..1a767e8 --- /dev/null +++ b/tests/bucket.py @@ -0,0 +1,89 @@ +#!/bin/env /usr/bin/python3 +# # Copyright (C) 2021 Aravinth Manivannan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from asyncio import sleep +import sys +import json + +from mcaptcha import register + +from dcache import grpc_add_vote, grpc_get_visitor_count + +def incr(key): + return grpc_add_vote(key) + +def get_count(key): + try: + count = grpc_get_visitor_count(key) + return int(count.visitors) + except: + return 0 + +def assert_count(expect, key): + count = get_count(key) + assert count == expect + +async def incr_one_works(): + try: + key = "incr_one" + register(key) + initial_count = get_count(key) + # incriment + incr(key) + assert_count(initial_count + 1, key) + # wait till expiry + await sleep(5 + 2) + assert_count(initial_count, key) + print("[*] Incr one works") + except Exception as e: + raise e + + +async def race_works(): + key = "race_works" + try: + register(key) + initial_count = get_count(key) + race_num = 200 + for _ in range(race_num): + incr(key) + assert_count(initial_count + race_num, key) + # wait till expiry + await sleep(5 + 2) + assert_count(initial_count, key) + print("[*] Race works") + except Exception as e: + raise e + + +async def difficulty_works(): + key = "difficulty_works" + try: + register(key) + data = incr(key) + assert data.difficulty_factor == 50 + + for _ in range(501): + incr(key) + data = incr(key) + assert data.difficulty_factor == 500 + + await sleep(5 + 2) + data = incr(key) + assert data.difficulty_factor == 50 + + print("[*] Difficulty factor works") + except Exception as e: + raise e diff --git a/tests/dcache.py b/tests/dcache.py new file mode 100644 index 0000000..e91bec2 --- /dev/null +++ b/tests/dcache.py @@ -0,0 +1,128 @@ +import requests +import grpc +import json + +from dcache_py import dcache_pb2 as dcache +from dcache_py.dcache_pb2 import RaftRequest +from dcache_py.dcache_pb2_grpc import DcacheServiceStub + +host = "localhost:9001" + +def grpc_add_vote(captcha_id: str): + with grpc.insecure_channel(host) as channel: + stub = DcacheServiceStub(channel) + + msg = dcache.CaptchaID(id=captcha_id) + resp = stub.AddVisitor(msg) + return resp.result + + +def grpc_add_captcha(captcha_id: str): + with grpc.insecure_channel(host) as channel: + stub = DcacheServiceStub(channel) + + msg = dcache.AddCaptchaRequest( + id=captcha_id, + mcaptcha=dcache.MCaptcha( + duration=5, + defense=dcache.Defense( + levels=[ + dcache.Level(visitor_threshold=50, difficulty_factor=50), + dcache.Level(visitor_threshold=500, difficulty_factor=500), + ] + ), + ), + ) + + + resp = stub.AddCaptcha(msg) + return resp + + +def grpc_captcha_exists(captcha_id: str): + with grpc.insecure_channel(host) as channel: + stub = DcacheServiceStub(channel) + msg = dcache.CaptchaID(id=captcha_id) + resp = stub.CaptchaExists(msg) + return resp.exists + + +def grpc_rename_captcha( captcha_id: str, new_id: str): + with grpc.insecure_channel(host) as channel: + stub = DcacheServiceStub(channel) + + msg = dcache.RenameCaptchaRequest(name=captcha_id, rename_to=new_id) + resp = stub.RenameCaptcha(msg) + + +def grpc_delete_captcha( captcha_id: str): + with grpc.insecure_channel(host) as channel: + stub = DcacheServiceStub(channel) + + msg = dcache.CaptchaID(id=captcha_id) + stub.RemoveCaptcha(msg) + + +def grpc_get_visitor_count(captcha_id: str): + with grpc.insecure_channel(host) as channel: + stub = DcacheServiceStub(channel) + msg = dcache.CaptchaID(id=captcha_id) + return stub.GetVisitorCount(msg).result + + +def grpc_add_challenge(token: str, key: str): + with grpc.insecure_channel(host) as channel: + stub = DcacheServiceStub(channel) + msg = dcache.CacheResultRequest( + token= token, + key= key, + duration = 5, + ) + stub.CacheResult(msg) + + +def grpc_get_challenge(token: str, key: str): + with grpc.insecure_channel(host) as channel: + stub = DcacheServiceStub(channel) + msg = dcache.RetrievePowRequest( + token= token, + key= key, + ) + return stub.VerifyCaptchaResult(msg) + +def grpc_delete_challenge(token: str): + with grpc.insecure_channel(host) as channel: + stub = DcacheServiceStub(channel) + msg = dcache.DeleteCaptchaResultRequest( + token= token, + ) + stub.DeleteCaptchaResult(msg) + +def grpc_add_pow(token: str, string: str): + with grpc.insecure_channel(host) as channel: + stub = DcacheServiceStub(channel) + msg = dcache.CachePowRequest( + key= token, + string= string, + duration = 5, + difficulty_factor= 500 + ) + return stub.CachePow(msg) + + +def grpc_get_pow(token: str, string: str): + with grpc.insecure_channel(host) as channel: + stub = DcacheServiceStub(channel) + msg = dcache.RetrievePowRequest( + token= string, + key= token) + resp= stub.RetrievePow(msg) + return resp + +def grpc_delete_pow(string: str): + with grpc.insecure_channel(host) as channel: + stub = DcacheServiceStub(channel) + msg = dcache.DeletePowRequest( + string= string, + ) + stub.DeletePow(msg) diff --git a/tests/mcaptcha.py b/tests/mcaptcha.py new file mode 100644 index 0000000..c073a29 --- /dev/null +++ b/tests/mcaptcha.py @@ -0,0 +1,80 @@ +#!/bin/env /usr/bin/python3 +# +# Copyright (C) 2021 Aravinth Manivannan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import json + +import utils +from dcache import grpc_add_captcha, grpc_add_vote, grpc_captcha_exists, grpc_rename_captcha +from dcache import grpc_delete_captcha + +from stub import get_stub + +def delete_captcha(key): + grpc_delete_captcha(key) + + +def add_captcha(key): + grpc_add_captcha(key) + +def rename_captcha(key, new_key): + grpc_rename_captcha(key, new_key) + + +def captcha_exists(key): + return grpc_captcha_exists(captcha_id=key) + +def register(key): + if captcha_exists(key): + delete_captcha(key) + add_captcha(key) + +async def captcha_exists_works(): + key = "captcha_delete_works" + if captcha_exists(key): + delete_captcha(key) + assert captcha_exists(key) is False + register(key) + assert captcha_exists(key) is True + print("[*] Captcha delete works") + +async def register_captcha_works(): + key = "register_captcha_works" + register(key) + assert captcha_exists(key) is True + print("[*] Add captcha works") + +async def delete_captcha_works(): + key = "delete_captcha_works" + register(key) + exists = captcha_exists(key) + assert exists is True + delete_captcha(key) + assert captcha_exists(key) is False + print("[*] Delete captcha works") + + +async def rename_captcha_works(): + key = "rename_captcha_works" + new_key = "new_key_rename_captcha_works" + register(key) + exists = captcha_exists(key) + assert exists is True + rename_captcha(key, new_key) + print(captcha_exists(key)) + assert captcha_exists(key) is False + assert captcha_exists(new_key) is True + print("[*] Rename captcha works") diff --git a/tests/pow.py b/tests/pow.py new file mode 100644 index 0000000..4071e92 --- /dev/null +++ b/tests/pow.py @@ -0,0 +1,115 @@ +#!/bin/env /usr/bin/python3 +# +# Copyright (C) 2023 Aravinth Manivannan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from asyncio import sleep +import json + +from dcache import grpc_add_pow, grpc_get_pow, grpc_delete_pow + +# 1. Check duplicate pow +# 2. Create pow +# 3. Read non-existent pow +# 4. Read pow +# 5. Read expired pow + +def add_pow(captcha, pow): + """Add pow to """ + try : + res = grpc_add_pow(captcha, pow) + return res + except Exception as e: + return e + +def get_pow_from(captcha, pow): + """Add pow to """ + try : + res = grpc_get_pow(captcha, pow) + if res.HasField('result'): + return res.result + else: + return None + except Exception as e: + return e + +def delete_pow(captcha, pow): + """Add pow to """ + try : + grpc_delete_pow(pow) + except Exception as e: + return e + +async def add_pow_works(): + """Test: Add pow""" + try: + key = "add_pow" + pow_name = "add_pow_pow" + + add_pow(key, pow_name) + stored_pow = get_pow_from(key, pow_name) + assert stored_pow.difficulty_factor == 500 + assert stored_pow.duration == 5 + print("[*] Add pow works") + + except Exception as e: + raise e + +async def pow_ttl_works(): + """Test: pow TTL""" + try: + key = "ttl_pow" + pow_name = "ttl_pow_pow" + + add_pow(key, pow_name) + await sleep(5 + 2) + + error = get_pow_from(key, pow_name) + assert error is None + + print("[*] pow TTL works") + except Exception as e: + raise e + + +async def pow_doesnt_exist(): + """Test: Non-existent pow""" + try: + pow_name = "nonexistent_pow" + key = "nonexistent_pow_key" + + error = get_pow_from(key, pow_name) + assert error is None + + print("[*] pow Doesn't Exist works") + except Exception as e: + raise e + + +async def delete_pow_works(): + """Test: Delete pows""" + try: + pow_name = "delete_pow" + key = "delete_pow_key" + #pow = get_pow(pow_name) + + add_pow(key, pow_name) + delete_pow(key, pow_name) + error = get_pow_from(key, pow_name) + assert error is None + + + print("[*] Delete pow works") + except Exception as e: + raise e diff --git a/tests/result.py b/tests/result.py new file mode 100644 index 0000000..dab452b --- /dev/null +++ b/tests/result.py @@ -0,0 +1,115 @@ +#!/bin/env /usr/bin/python3 +# +# Copyright (C) 2023 Aravinth Manivannan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from asyncio import sleep +import json + +import utils +from dcache import grpc_add_challenge, grpc_get_challenge, grpc_delete_challenge + +# 1. Check duplicate result +# 2. Create result +# 3. Read non-existent result +# 4. Read result +# 5. Read expired result + + +COMMANDS = { + "ADD" :"MCAPTCHA_CACHE.ADD_result", + "GET" :"MCAPTCHA_CACHE.GET_result", + "DEL" :"MCAPTCHA_CACHE.DELETE_result" +} + +result_NOT_FOUND = "result not found" +DUPLICATE_result = "result already exists" +REDIS_OK = bytes("OK", 'utf-8') + +def add_result(captcha, result): + """Add result to """ + try : + grpc_add_challenge(captcha,result) + except Exception as e: + return e + +def get_result_from(captcha, result): + """Add result to """ + try : + return grpc_get_challenge(captcha,result) + except Exception as e: + return e + +def delete_result(captcha, result): + """Add result to """ + try : + grpc_delete_challenge(captcha) + except Exception as e: + return e + +async def add_result_works(): + """Test: Add result""" + try: + key = "add_result" + result_name = "add_result_result" + + add_result(key, result_name) + verified = get_result_from(key, result_name) + assert verified.verified is True + print("[*] Add result works") + + except Exception as e: + raise e + +async def result_ttl_works(): + """Test: result TTL""" + try: + key = "ttl_result" + result_name = "ttl_result_result" + + add_result(key, result_name) + await sleep(5 + 2) + + error = get_result_from(key, result_name) +# assert str(error) == result_NOT_FOUND + + print("[*] result TTL works") + except Exception as e: + raise e + + +async def result_doesnt_exist(): + """Test: Non-existent result""" + try: + result_name = "nonexistent_result" + key = "nonexistent_result_key" + + error = get_result_from(key, result_name) + print("[*] result Doesn't Exist works") + except Exception as e: + raise e + + +async def delete_result_works(): + """Test: Delete results""" + try: + result_name = "delete_result" + key = "delete_result_key" + + add_result(key, result_name) + resp = delete_result(key, result_name) + + print("[*] Delete result works") + except Exception as e: + raise e diff --git a/tests/runner.py b/tests/runner.py new file mode 100644 index 0000000..7c511ff --- /dev/null +++ b/tests/runner.py @@ -0,0 +1,63 @@ +#!/bin/env /usr/bin/python3 +# Copyright (C) 2023 Aravinth Manivannan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from threading import Thread +import asyncio + +import importlib.util +import sys +sys.path.append('/home/atm/code/mcaptcha/dcache/') + +import bucket +import mcaptcha +import result +import pow + +class Runner(object): + __fn = [ + bucket.incr_one_works, + bucket.race_works, + bucket.difficulty_works, + mcaptcha.delete_captcha_works, + mcaptcha.captcha_exists_works, + mcaptcha.register_captcha_works, + mcaptcha.rename_captcha_works, + result.add_result_works, + result.result_doesnt_exist, + result.result_ttl_works, + result.delete_result_works, + pow.add_pow_works, + pow.pow_doesnt_exist, + pow.pow_ttl_works, + pow.delete_pow_works, + ] + __tasks = [] + + async def __register(self): + """ Register functions to be run""" + for fn in self.__fn: + task = asyncio.create_task(fn()) + self.__tasks.append(task) + + + async def run(self): + """Wait for registered functions to finish executing""" + await self.__register() + for task in self.__tasks: + await task + + """Runs in separate threads""" + def __init__(self): + super(Runner, self).__init__() diff --git a/tests/test.py b/tests/test.py new file mode 100755 index 0000000..b629da5 --- /dev/null +++ b/tests/test.py @@ -0,0 +1,29 @@ +#!/bin/env /usr/bin/python3 +# +# Copyright (C) 2021 Aravinth Manivannan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +import asyncio + +from runner import Runner + + +async def main(): + print("Running Integration Tests") + runner = Runner() + await runner.run() + print("All tests passed") + +if __name__ == "__main__": + asyncio.run(main()) From e548a532a036298f0e7cdbfd82fc18a0cfe5b0ed Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Sun, 31 Dec 2023 01:28:43 +0530 Subject: [PATCH 2/9] feat: add integration testing --- .woodpecker.yml | 12 ++++++++++++ requirements.txt | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 requirements.txt diff --git a/.woodpecker.yml b/.woodpecker.yml index 197b8b3..5b485d6 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -5,10 +5,22 @@ steps: - apt update - apt-get install -y --no-install-recommends protobuf-compiler - cargo build + - cargo test # - make migrate # - make # - make release # - make test // requires Docker-in-Docker + backend: + image: python + commands: + - pip install -r requirements.txt + - nohup ./target/debug/main + - nohup ./target/debug/main --id 1 --http-addr 127.0.0.1:9001 --introducer-addr 127.0.0.1:9001 --introducer-id 1 --cluster-size 3 & + - sleep 1 + - nohup ./target/debug/main --id 2 --http-addr 127.0.0.1:9002 --introducer-addr 127.0.0.1:9001 --introducer-id 1 --cluster-size 3 & + - sleep 1 + - nohup ./target/debug/main --id 3 --http-addr 127.0.0.1:9003 --introducer-addr 127.0.0.1:9001 --introducer-id 1 --cluster-size 3 & + - python tests/tests.py build_docker_img: image: plugins/docker diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..97bbe09 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,32 @@ +asyncio==3.4.3 +blinker==1.7.0 +Brotli==1.1.0 +certifi==2023.11.17 +charset-normalizer==3.3.2 +click==8.1.7 +ConfigArgParse==1.7 +Flask==3.0.0 +Flask-BasicAuth==0.2.0 +Flask-Cors==4.0.0 +gevent==23.9.1 +geventhttpclient==2.0.11 +greenlet==3.0.2 +grpc-interceptor==0.15.4 +grpcio==1.60.0 +grpcio-tools==1.60.0 +idna==3.6 +itsdangerous==2.1.2 +Jinja2==3.1.2 +locust==2.20.0 +MarkupSafe==2.1.3 +msgpack==1.0.7 +protobuf==4.25.1 +psutil==5.9.7 +pyzmq==25.1.2 +requests==2.31.0 +roundrobin==0.0.4 +six==1.16.0 +urllib3==2.1.0 +Werkzeug==3.0.1 +zope.event==5.0 +zope.interface==6.1 From 240b5ec13ad42c846b6498cf83cabf6a47f7a316 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Sun, 31 Dec 2023 01:28:59 +0530 Subject: [PATCH 3/9] chore: linting --- tests/bucket.py | 12 ++++++++---- tests/dcache.py | 44 +++++++++++++++++++++----------------------- tests/mcaptcha.py | 19 +++++++++++++++---- tests/pow.py | 30 +++++++++++++++++------------- tests/result.py | 45 +++++++++++++++++++++++++-------------------- tests/runner.py | 14 ++++++++------ tests/test.py | 7 ++++--- 7 files changed, 98 insertions(+), 73 deletions(-) diff --git a/tests/bucket.py b/tests/bucket.py index 1a767e8..b83eb4a 100644 --- a/tests/bucket.py +++ b/tests/bucket.py @@ -1,16 +1,16 @@ #!/bin/env /usr/bin/python3 # # Copyright (C) 2021 Aravinth Manivannan -# +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. -# +# # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . from asyncio import sleep @@ -21,9 +21,11 @@ from mcaptcha import register from dcache import grpc_add_vote, grpc_get_visitor_count + def incr(key): return grpc_add_vote(key) + def get_count(key): try: count = grpc_get_visitor_count(key) @@ -31,10 +33,12 @@ def get_count(key): except: return 0 + def assert_count(expect, key): count = get_count(key) assert count == expect + async def incr_one_works(): try: key = "incr_one" @@ -74,7 +78,7 @@ async def difficulty_works(): register(key) data = incr(key) assert data.difficulty_factor == 50 - + for _ in range(501): incr(key) data = incr(key) diff --git a/tests/dcache.py b/tests/dcache.py index e91bec2..a65d548 100644 --- a/tests/dcache.py +++ b/tests/dcache.py @@ -8,6 +8,7 @@ from dcache_py.dcache_pb2_grpc import DcacheServiceStub host = "localhost:9001" + def grpc_add_vote(captcha_id: str): with grpc.insecure_channel(host) as channel: stub = DcacheServiceStub(channel) @@ -34,7 +35,6 @@ def grpc_add_captcha(captcha_id: str): ), ) - resp = stub.AddCaptcha(msg) return resp @@ -47,7 +47,7 @@ def grpc_captcha_exists(captcha_id: str): return resp.exists -def grpc_rename_captcha( captcha_id: str, new_id: str): +def grpc_rename_captcha(captcha_id: str, new_id: str): with grpc.insecure_channel(host) as channel: stub = DcacheServiceStub(channel) @@ -55,7 +55,7 @@ def grpc_rename_captcha( captcha_id: str, new_id: str): resp = stub.RenameCaptcha(msg) -def grpc_delete_captcha( captcha_id: str): +def grpc_delete_captcha(captcha_id: str): with grpc.insecure_channel(host) as channel: stub = DcacheServiceStub(channel) @@ -74,10 +74,10 @@ def grpc_add_challenge(token: str, key: str): with grpc.insecure_channel(host) as channel: stub = DcacheServiceStub(channel) msg = dcache.CacheResultRequest( - token= token, - key= key, - duration = 5, - ) + token=token, + key=key, + duration=5, + ) stub.CacheResult(msg) @@ -85,44 +85,42 @@ def grpc_get_challenge(token: str, key: str): with grpc.insecure_channel(host) as channel: stub = DcacheServiceStub(channel) msg = dcache.RetrievePowRequest( - token= token, - key= key, - ) + token=token, + key=key, + ) return stub.VerifyCaptchaResult(msg) + def grpc_delete_challenge(token: str): with grpc.insecure_channel(host) as channel: stub = DcacheServiceStub(channel) msg = dcache.DeleteCaptchaResultRequest( - token= token, - ) + token=token, + ) stub.DeleteCaptchaResult(msg) + def grpc_add_pow(token: str, string: str): with grpc.insecure_channel(host) as channel: stub = DcacheServiceStub(channel) msg = dcache.CachePowRequest( - key= token, - string= string, - duration = 5, - difficulty_factor= 500 - ) + key=token, string=string, duration=5, difficulty_factor=500 + ) return stub.CachePow(msg) def grpc_get_pow(token: str, string: str): with grpc.insecure_channel(host) as channel: stub = DcacheServiceStub(channel) - msg = dcache.RetrievePowRequest( - token= string, - key= token) - resp= stub.RetrievePow(msg) + msg = dcache.RetrievePowRequest(token=string, key=token) + resp = stub.RetrievePow(msg) return resp + def grpc_delete_pow(string: str): with grpc.insecure_channel(host) as channel: stub = DcacheServiceStub(channel) msg = dcache.DeletePowRequest( - string= string, - ) + string=string, + ) stub.DeletePow(msg) diff --git a/tests/mcaptcha.py b/tests/mcaptcha.py index c073a29..63ad51a 100644 --- a/tests/mcaptcha.py +++ b/tests/mcaptcha.py @@ -1,28 +1,34 @@ #!/bin/env /usr/bin/python3 # # Copyright (C) 2021 Aravinth Manivannan -# +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. -# +# # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . import json import utils -from dcache import grpc_add_captcha, grpc_add_vote, grpc_captcha_exists, grpc_rename_captcha +from dcache import ( + grpc_add_captcha, + grpc_add_vote, + grpc_captcha_exists, + grpc_rename_captcha, +) from dcache import grpc_delete_captcha from stub import get_stub + def delete_captcha(key): grpc_delete_captcha(key) @@ -30,6 +36,7 @@ def delete_captcha(key): def add_captcha(key): grpc_add_captcha(key) + def rename_captcha(key, new_key): grpc_rename_captcha(key, new_key) @@ -37,11 +44,13 @@ def rename_captcha(key, new_key): def captcha_exists(key): return grpc_captcha_exists(captcha_id=key) + def register(key): if captcha_exists(key): delete_captcha(key) add_captcha(key) + async def captcha_exists_works(): key = "captcha_delete_works" if captcha_exists(key): @@ -51,12 +60,14 @@ async def captcha_exists_works(): assert captcha_exists(key) is True print("[*] Captcha delete works") + async def register_captcha_works(): key = "register_captcha_works" register(key) assert captcha_exists(key) is True print("[*] Add captcha works") + async def delete_captcha_works(): key = "delete_captcha_works" register(key) diff --git a/tests/pow.py b/tests/pow.py index 4071e92..6cd37e3 100644 --- a/tests/pow.py +++ b/tests/pow.py @@ -1,17 +1,17 @@ #!/bin/env /usr/bin/python3 # # Copyright (C) 2023 Aravinth Manivannan -# +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. -# +# # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . from asyncio import sleep @@ -25,32 +25,36 @@ from dcache import grpc_add_pow, grpc_get_pow, grpc_delete_pow # 4. Read pow # 5. Read expired pow + def add_pow(captcha, pow): - """Add pow to """ - try : + """Add pow to""" + try: res = grpc_add_pow(captcha, pow) return res except Exception as e: return e - + + def get_pow_from(captcha, pow): - """Add pow to """ - try : + """Add pow to""" + try: res = grpc_get_pow(captcha, pow) - if res.HasField('result'): + if res.HasField("result"): return res.result else: return None except Exception as e: return e + def delete_pow(captcha, pow): - """Add pow to """ - try : + """Add pow to""" + try: grpc_delete_pow(pow) except Exception as e: return e + async def add_pow_works(): """Test: Add pow""" try: @@ -66,6 +70,7 @@ async def add_pow_works(): except Exception as e: raise e + async def pow_ttl_works(): """Test: pow TTL""" try: @@ -102,14 +107,13 @@ async def delete_pow_works(): try: pow_name = "delete_pow" key = "delete_pow_key" - #pow = get_pow(pow_name) + # pow = get_pow(pow_name) add_pow(key, pow_name) delete_pow(key, pow_name) error = get_pow_from(key, pow_name) assert error is None - print("[*] Delete pow works") except Exception as e: raise e diff --git a/tests/result.py b/tests/result.py index dab452b..9ec58be 100644 --- a/tests/result.py +++ b/tests/result.py @@ -1,17 +1,17 @@ #!/bin/env /usr/bin/python3 # # Copyright (C) 2023 Aravinth Manivannan -# +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. -# +# # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . from asyncio import sleep @@ -28,36 +28,40 @@ from dcache import grpc_add_challenge, grpc_get_challenge, grpc_delete_challenge COMMANDS = { - "ADD" :"MCAPTCHA_CACHE.ADD_result", - "GET" :"MCAPTCHA_CACHE.GET_result", - "DEL" :"MCAPTCHA_CACHE.DELETE_result" + "ADD": "MCAPTCHA_CACHE.ADD_result", + "GET": "MCAPTCHA_CACHE.GET_result", + "DEL": "MCAPTCHA_CACHE.DELETE_result", } result_NOT_FOUND = "result not found" DUPLICATE_result = "result already exists" -REDIS_OK = bytes("OK", 'utf-8') +REDIS_OK = bytes("OK", "utf-8") + def add_result(captcha, result): - """Add result to """ - try : - grpc_add_challenge(captcha,result) - except Exception as e: - return e - -def get_result_from(captcha, result): - """Add result to """ - try : - return grpc_get_challenge(captcha,result) + """Add result to""" + try: + grpc_add_challenge(captcha, result) except Exception as e: return e + +def get_result_from(captcha, result): + """Add result to""" + try: + return grpc_get_challenge(captcha, result) + except Exception as e: + return e + + def delete_result(captcha, result): - """Add result to """ - try : + """Add result to""" + try: grpc_delete_challenge(captcha) except Exception as e: return e + async def add_result_works(): """Test: Add result""" try: @@ -72,6 +76,7 @@ async def add_result_works(): except Exception as e: raise e + async def result_ttl_works(): """Test: result TTL""" try: @@ -82,7 +87,7 @@ async def result_ttl_works(): await sleep(5 + 2) error = get_result_from(key, result_name) -# assert str(error) == result_NOT_FOUND + # assert str(error) == result_NOT_FOUND print("[*] result TTL works") except Exception as e: diff --git a/tests/runner.py b/tests/runner.py index 7c511ff..b8ac110 100644 --- a/tests/runner.py +++ b/tests/runner.py @@ -1,16 +1,16 @@ #!/bin/env /usr/bin/python3 # Copyright (C) 2023 Aravinth Manivannan -# +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. -# +# # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . from threading import Thread @@ -18,13 +18,15 @@ import asyncio import importlib.util import sys -sys.path.append('/home/atm/code/mcaptcha/dcache/') + +sys.path.append("/home/atm/code/mcaptcha/dcache/") import bucket import mcaptcha import result import pow + class Runner(object): __fn = [ bucket.incr_one_works, @@ -46,12 +48,11 @@ class Runner(object): __tasks = [] async def __register(self): - """ Register functions to be run""" + """Register functions to be run""" for fn in self.__fn: task = asyncio.create_task(fn()) self.__tasks.append(task) - async def run(self): """Wait for registered functions to finish executing""" await self.__register() @@ -59,5 +60,6 @@ class Runner(object): await task """Runs in separate threads""" + def __init__(self): super(Runner, self).__init__() diff --git a/tests/test.py b/tests/test.py index b629da5..0807458 100755 --- a/tests/test.py +++ b/tests/test.py @@ -1,17 +1,17 @@ #!/bin/env /usr/bin/python3 # # Copyright (C) 2021 Aravinth Manivannan -# +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. -# +# # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . import asyncio @@ -25,5 +25,6 @@ async def main(): await runner.run() print("All tests passed") + if __name__ == "__main__": asyncio.run(main()) From 73aa7550356b7baa1995f2623e77ad285196aaf1 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Sun, 31 Dec 2023 01:38:59 +0530 Subject: [PATCH 4/9] fix: CI: use different stage name for integration tests --- .woodpecker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 5b485d6..c4f66e7 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -5,12 +5,12 @@ steps: - apt update - apt-get install -y --no-install-recommends protobuf-compiler - cargo build - - cargo test + - cargo test --lib # - make migrate # - make # - make release # - make test // requires Docker-in-Docker - backend: + integration_tests: image: python commands: - pip install -r requirements.txt From ae6651e6247da99322a5a371d20aa1323e11a207 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Sun, 31 Dec 2023 01:46:44 +0530 Subject: [PATCH 5/9] fix: rm stray command --- .woodpecker.yml | 3 +-- tests/mcaptcha.py | 1 - tests/result.py | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index c4f66e7..d016b3e 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -14,13 +14,12 @@ steps: image: python commands: - pip install -r requirements.txt - - nohup ./target/debug/main - nohup ./target/debug/main --id 1 --http-addr 127.0.0.1:9001 --introducer-addr 127.0.0.1:9001 --introducer-id 1 --cluster-size 3 & - sleep 1 - nohup ./target/debug/main --id 2 --http-addr 127.0.0.1:9002 --introducer-addr 127.0.0.1:9001 --introducer-id 1 --cluster-size 3 & - sleep 1 - nohup ./target/debug/main --id 3 --http-addr 127.0.0.1:9003 --introducer-addr 127.0.0.1:9001 --introducer-id 1 --cluster-size 3 & - - python tests/tests.py + - python tests/test.py build_docker_img: image: plugins/docker diff --git a/tests/mcaptcha.py b/tests/mcaptcha.py index 63ad51a..f1ee5bb 100644 --- a/tests/mcaptcha.py +++ b/tests/mcaptcha.py @@ -17,7 +17,6 @@ import json -import utils from dcache import ( grpc_add_captcha, grpc_add_vote, diff --git a/tests/result.py b/tests/result.py index 9ec58be..af90703 100644 --- a/tests/result.py +++ b/tests/result.py @@ -17,7 +17,6 @@ from asyncio import sleep import json -import utils from dcache import grpc_add_challenge, grpc_get_challenge, grpc_delete_challenge # 1. Check duplicate result From f47c0867d339d05f3caee9b3cff175abb4fdcc44 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Sun, 31 Dec 2023 02:37:01 +0530 Subject: [PATCH 6/9] fix: use python virtualenv --- .woodpecker.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index d016b3e..63031ab 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -13,13 +13,14 @@ steps: integration_tests: image: python commands: - - pip install -r requirements.txt + - pip install virtualenv + - . venv/bin/activate && pip install -r requirements.txt - nohup ./target/debug/main --id 1 --http-addr 127.0.0.1:9001 --introducer-addr 127.0.0.1:9001 --introducer-id 1 --cluster-size 3 & - sleep 1 - nohup ./target/debug/main --id 2 --http-addr 127.0.0.1:9002 --introducer-addr 127.0.0.1:9001 --introducer-id 1 --cluster-size 3 & - sleep 1 - nohup ./target/debug/main --id 3 --http-addr 127.0.0.1:9003 --introducer-addr 127.0.0.1:9001 --introducer-id 1 --cluster-size 3 & - - python tests/test.py + - . venv/bin/activate && python tests/test.py build_docker_img: image: plugins/docker From a10fb878f53e7298d0be0ce467249c0a959ae82c Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Sun, 31 Dec 2023 03:03:00 +0530 Subject: [PATCH 7/9] fix: rm stub --- tests/mcaptcha.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/mcaptcha.py b/tests/mcaptcha.py index f1ee5bb..a453a5f 100644 --- a/tests/mcaptcha.py +++ b/tests/mcaptcha.py @@ -25,9 +25,6 @@ from dcache import ( ) from dcache import grpc_delete_captcha -from stub import get_stub - - def delete_captcha(key): grpc_delete_captcha(key) From d7fe9332d6afddfb5e84b90318006e9ce7ef9f27 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Sun, 31 Dec 2023 03:04:34 +0530 Subject: [PATCH 8/9] debug: mv dcache_py into tess --- .woodpecker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.woodpecker.yml b/.woodpecker.yml index 63031ab..fa95bc3 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -20,6 +20,7 @@ steps: - nohup ./target/debug/main --id 2 --http-addr 127.0.0.1:9002 --introducer-addr 127.0.0.1:9001 --introducer-id 1 --cluster-size 3 & - sleep 1 - nohup ./target/debug/main --id 3 --http-addr 127.0.0.1:9003 --introducer-addr 127.0.0.1:9001 --introducer-id 1 --cluster-size 3 & + - mv dcache_py/ tests/ - . venv/bin/activate && python tests/test.py build_docker_img: From ff71e35da3c56746631ff6d34cb246214a7bab86 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Sun, 31 Dec 2023 03:05:06 +0530 Subject: [PATCH 9/9] fix: create virtualenv --- .woodpecker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index fa95bc3..79491f8 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -13,7 +13,7 @@ steps: integration_tests: image: python commands: - - pip install virtualenv + - pip install virtualenv && virtualenv venv - . venv/bin/activate && pip install -r requirements.txt - nohup ./target/debug/main --id 1 --http-addr 127.0.0.1:9001 --introducer-addr 127.0.0.1:9001 --introducer-id 1 --cluster-size 3 & - sleep 1