feat: actix-web and tokio_tungstenite based webrtc impl

Cargo.lock

@@ -10,7 +10,7 @@ checksum = "f728064aca1c318585bf4bb04ffcfac9e75e508ab4e8b1bd9ba5dfe04e2cbed5"
 dependencies = [
  "actix-rt",
  "actix_derive",
- "bitflags",
+ "bitflags 1.3.2",
  "bytes",
  "crossbeam-channel",
  "futures-core",
@@ -32,7 +32,7 @@ version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
  "bytes",
  "futures-core",
  "futures-sink",
@@ -45,9 +45,9 @@ dependencies = [
 
 [[package]]
 name = "actix-http"
-version = "3.3.1"
+version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2079246596c18b4a33e274ae10c0e50613f4d32a4198e09c7b93771013fed74"
+checksum = "a92ef85799cba03f76e4f7c10f533e66d87c9a7e7055f3391f09000ad8351bc9"
 dependencies = [
  "actix-codec",
  "actix-rt",
@@ -55,7 +55,7 @@ dependencies = [
  "actix-utils",
  "ahash 0.8.3",
  "base64 0.21.1",
- "bitflags",
+ "bitflags 2.4.1",
  "brotli",
  "bytes",
  "bytestring",
@@ -64,7 +64,7 @@ dependencies = [
  "flate2",
  "futures-core",
  "h2",
- "http",
+ "http 0.2.9",
  "httparse",
  "httpdate",
  "itoa",
@@ -99,7 +99,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d66ff4d247d2b160861fa2866457e85706833527840e4133f8f49aa423a38799"
 dependencies = [
  "bytestring",
- "http",
+ "http 0.2.9",
  "regex",
  "serde 1.0.163",
  "tracing",
@@ -145,6 +145,26 @@ dependencies = [
  "pin-project-lite",
 ]
 
+[[package]]
+name = "actix-tls"
+version = "3.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72616e7fbec0aa99c6f3164677fa48ff5a60036d0799c98cab894a44f3e0efc3"
+dependencies = [
+ "actix-rt",
+ "actix-service",
+ "actix-utils",
+ "futures-core",
+ "http 0.2.9",
+ "impl-more",
+ "pin-project-lite",
+ "rustls",
+ "rustls-webpki",
+ "tokio",
+ "tokio-util",
+ "tracing",
+]
+
 [[package]]
 name = "actix-utils"
 version = "3.0.1"
@@ -179,7 +199,7 @@ dependencies = [
  "encoding_rs",
  "futures-core",
  "futures-util",
- "http",
+ "http 0.2.9",
  "itoa",
  "language-tags",
  "log",
@@ -196,6 +216,24 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "actix-web-actors"
+version = "4.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf6e9ccc371cfddbed7aa842256a4abc7a6dcac9f3fce392fe1d0f68cfd136b2"
+dependencies = [
+ "actix",
+ "actix-codec",
+ "actix-http",
+ "actix-web",
+ "bytes",
+ "bytestring",
+ "futures-core",
+ "pin-project-lite",
+ "tokio",
+ "tokio-util",
+]
+
 [[package]]
 name = "actix-web-codegen"
 version = "4.2.0"
@@ -402,6 +440,39 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
+[[package]]
+name = "awc"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fa3c705a9c7917ac0f41c0757a0a747b43bbc29b0b364b081bd7c5fc67fb223"
+dependencies = [
+ "actix-codec",
+ "actix-http",
+ "actix-rt",
+ "actix-service",
+ "actix-tls",
+ "actix-utils",
+ "base64 0.21.1",
+ "bytes",
+ "cfg-if",
+ "cookie",
+ "derive_more",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http 0.2.9",
+ "itoa",
+ "log",
+ "mime",
+ "percent-encoding",
+ "pin-project-lite",
+ "rand",
+ "serde 1.0.163",
+ "serde_json",
+ "serde_urlencoded",
+ "tokio",
+]
+
 [[package]]
 name = "base64"
 version = "0.13.1"
@@ -429,6 +500,12 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "bitflags"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
+
 [[package]]
 name = "block-buffer"
 version = "0.10.4"
@@ -530,7 +607,7 @@ checksum = "4f423e341edefb78c9caba2d9c7f7687d0e72e89df3ce3394554754393ac3990"
 dependencies = [
  "anstream",
  "anstyle",
- "bitflags",
+ "bitflags 1.3.2",
  "clap_lex",
  "strsim",
 ]
@@ -710,23 +787,33 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "data-encoding"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5"
+
 [[package]]
 name = "dcache"
 version = "0.1.0"
 dependencies = [
  "actix",
+ "actix-codec",
  "actix-rt",
  "actix-web",
+ "actix-web-actors",
  "actix-web-codegen-const-routes",
  "actix-web-httpauth",
  "anyhow",
  "async-trait",
+ "awc",
  "base64 0.13.1",
  "byteorder",
  "clap",
  "config",
  "derive_builder 0.11.2",
  "derive_more",
+ "futures",
  "futures-util",
  "lazy_static",
  "libmcaptcha",
@@ -737,6 +824,7 @@ dependencies = [
  "serde 1.0.163",
  "serde_json",
  "tokio",
+ "tokio-tungstenite",
  "tracing",
  "tracing-subscriber",
  "url",
@@ -935,9 +1023,9 @@ dependencies = [
 
 [[package]]
 name = "futures"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40"
+checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -950,9 +1038,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2"
+checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -960,15 +1048,15 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c"
+checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c"
 
 [[package]]
 name = "futures-executor"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0"
+checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc"
 dependencies = [
  "futures-core",
  "futures-task",
@@ -977,15 +1065,15 @@ dependencies = [
 
 [[package]]
 name = "futures-io"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"
+checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa"
 
 [[package]]
 name = "futures-macro"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
+checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -994,21 +1082,21 @@ dependencies = [
 
 [[package]]
 name = "futures-sink"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e"
+checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817"
 
 [[package]]
 name = "futures-task"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65"
+checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2"
 
 [[package]]
 name = "futures-util"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
+checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -1054,7 +1142,7 @@ dependencies = [
  "futures-core",
  "futures-sink",
  "futures-util",
- "http",
+ "http 0.2.9",
  "indexmap",
  "slab",
  "tokio",
@@ -1109,6 +1197,17 @@ dependencies = [
  "itoa",
 ]
 
+[[package]]
+name = "http"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea"
+dependencies = [
+ "bytes",
+ "fnv",
+ "itoa",
+]
+
 [[package]]
 name = "http-body"
 version = "0.4.5"
@@ -1116,7 +1215,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
 dependencies = [
  "bytes",
- "http",
+ "http 0.2.9",
  "pin-project-lite",
 ]
 
@@ -1158,7 +1257,7 @@ dependencies = [
  "futures-core",
  "futures-util",
  "h2",
- "http",
+ "http 0.2.9",
  "http-body",
  "httparse",
  "httpdate",
@@ -1200,6 +1299,12 @@ dependencies = [
  "unicode-normalization",
 ]
 
+[[package]]
+name = "impl-more"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "206ca75c9c03ba3d4ace2460e57b189f39f43de612c2f85836e65c929701bb2d"
+
 [[package]]
 name = "indexmap"
 version = "1.9.3"
@@ -1291,7 +1396,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe"
 dependencies = [
  "arrayvec",
- "bitflags",
+ "bitflags 1.3.2",
  "cfg-if",
  "ryu",
  "static_assertions",
@@ -1603,7 +1708,7 @@ version = "0.10.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "01b8574602df80f7b85fdfc5392fa884a4e3b3f4f35402c070ab34c3d3f78d56"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
  "cfg-if",
  "foreign-types",
  "libc",
@@ -1748,9 +1853,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.58"
+version = "1.0.70"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8"
+checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b"
 dependencies = [
  "unicode-ident",
 ]
@@ -1844,7 +1949,7 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 ]
 
 [[package]]
@@ -1853,7 +1958,7 @@ version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 ]
 
 [[package]]
@@ -1900,7 +2005,7 @@ dependencies = [
  "futures-core",
  "futures-util",
  "h2",
- "http",
+ "http 0.2.9",
  "http-body",
  "hyper",
  "hyper-tls",
@@ -1925,6 +2030,20 @@ dependencies = [
  "winreg",
 ]
 
+[[package]]
+name = "ring"
+version = "0.17.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9babe80d5c16becf6594aa32ad2be8fe08498e7ae60b77de8df700e67f191d7e"
+dependencies = [
+ "cc",
+ "getrandom",
+ "libc",
+ "spin",
+ "untrusted",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "rust-ini"
 version = "0.13.0"
@@ -1946,7 +2065,7 @@ version = "0.37.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
  "errno",
  "io-lifetimes",
  "libc",
@@ -1954,6 +2073,28 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "rustls"
+version = "0.21.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba"
+dependencies = [
+ "log",
+ "ring",
+ "rustls-webpki",
+ "sct",
+]
+
+[[package]]
+name = "rustls-webpki"
+version = "0.101.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
 [[package]]
 name = "ryu"
 version = "1.0.13"
@@ -1984,13 +2125,23 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "sct"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
 [[package]]
 name = "security-framework"
 version = "2.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
  "core-foundation",
  "core-foundation-sys",
  "libc",
@@ -2145,6 +2296,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "spin"
+version = "0.9.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+
 [[package]]
 name = "static_assertions"
 version = "1.1.0"
@@ -2324,6 +2481,18 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-tungstenite"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38"
+dependencies = [
+ "futures-util",
+ "log",
+ "tokio",
+ "tungstenite",
+]
+
 [[package]]
 name = "tokio-util"
 version = "0.7.8"
@@ -2432,6 +2601,25 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
 
+[[package]]
+name = "tungstenite"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1"
+dependencies = [
+ "byteorder",
+ "bytes",
+ "data-encoding",
+ "http 1.0.0",
+ "httparse",
+ "log",
+ "rand",
+ "sha1",
+ "thiserror",
+ "url",
+ "utf-8",
+]
+
 [[package]]
 name = "typenum"
 version = "1.16.0"
@@ -2459,6 +2647,12 @@ dependencies = [
  "tinyvec",
 ]
 
+[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
 [[package]]
 name = "url"
 version = "2.3.1"
@@ -2471,6 +2665,12 @@ dependencies = [
  "serde 1.0.163",
 ]
 
+[[package]]
+name = "utf-8"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
+
 [[package]]
 name = "utf8-width"
 version = "0.1.6"

Cargo.toml

@@ -32,6 +32,11 @@ reqwest = { version = "0.11.9", features = ["json"] }
 tokio = { version = "1.0", default-features = false, features = ["sync"] }
 tracing-subscriber = { version = "0.3.0", features = ["env-filter"] }
 actix = "0.13.0"
+actix-web-actors = "4.2.0"
+awc = "3.2.0"
+futures = "0.3.29"
+actix-codec = "0.5.1"
+tokio-tungstenite = "0.21.0"
 
 
 [build-dependencies]

bench/locustfile.py

@@ -16,9 +16,9 @@ class Unprotected(FastHttpUser):
     host = leader
     captcha_id="locust"
 
-    pipeline_vote = []
+#    pipeline_vote = []
-    for _ in range(0,1000):
+#    for _ in range(0,100):
-        pipeline_vote.append({"AddVisitor": captcha_id})
+#        pipeline_vote.append({"AddVisitor": captcha_id})
 
 #    def on_start(self):
 #        resp  = self.client.get(f"{self.leader}/metrics")
@@ -33,7 +33,7 @@ class Unprotected(FastHttpUser):
 
     def write(self, data):
         resp = self.client.post(f"{self.host}/write", json=data)
-        print(f"RPC Status: {resp.status_code}")
+#        print(f"RPC Status: {resp.status_code}")
         resp = resp.json()
         if "Err" in resp:
             leader = resp["Err"]["APIError"]["ForwardToLeader"]["leader_node"]["addr"]
@@ -56,7 +56,7 @@ class Unprotected(FastHttpUser):
 
     def add_vote(self, captcha_id: str):
         resp = self.write(data={"AddVisitor": captcha_id})
-        pprint(resp)
+    #    pprint(resp)
 
     def add_vote_pipeline(self, captcha_id: str):
         resp = self.pipeline_write(data=self.pipeline_vote)
@@ -85,8 +85,8 @@ class Unprotected(FastHttpUser):
 
     @task
     def unprotected(self):
-        self.add_vote_pipeline(captcha_id=self.captcha_id)
+        #self.add_vote_pipeline(captcha_id=self.captcha_id)
-        ##self.add_vote(captcha_id="locust")
+        self.add_vote(captcha_id="locust")
 #        data = {
 #            "username": username,
 #            "password": username,

bench/results/v1/individual/exceptions_1702823615.6765826.csv

@@ -0,0 +1 @@
+Count,Message,Traceback,Nodes

bench/results/v1/individual/failures_1702823611.147026.csv

@@ -0,0 +1 @@
+Method,Name,Error,Occurrences

bench/results/v1/individual/requests_1702823585.0296123.csv

@@ -0,0 +1,3 @@
+Type,Name,Request Count,Failure Count,Median Response Time,Average Response Time,Min Response Time,Max Response Time,Average Content Size,Requests/s,Failures/s,50%,66%,75%,80%,90%,95%,98%,99%,99.9%,99.99%,100%
+POST,http://localhost:9001/write,348265,0,130,133.02968716351054,0,544,98.68140640029863,2517.2083812580477,0.0,130,160,180,190,310,350,370,390,420,530,540
+,Aggregated,348265,0,130,133.02968716351054,0,544,98.68140640029863,2517.2083812580477,0.0,130,160,180,190,310,350,370,390,420,530,540

bench/results/v1/nopooling-nopipeline/exceptions_1702930188.249846.csv

@@ -0,0 +1 @@
+Count,Message,Traceback,Nodes

bench/results/v1/nopooling-nopipeline/failures_1702930183.0106597.csv

@@ -0,0 +1 @@
+Method,Name,Error,Occurrences

bench/results/v1/nopooling-nopipeline/requests_1702930173.98783.csv

@@ -0,0 +1,3 @@
+Type,Name,Request Count,Failure Count,Median Response Time,Average Response Time,Min Response Time,Max Response Time,Average Content Size,Requests/s,Failures/s,50%,66%,75%,80%,90%,95%,98%,99%,99.9%,99.99%,100%
+POST,http://localhost:9001/write,47348,0,350,344.40586719608007,5,1034,97.76577680155445,993.0375320581431,0.0,350,390,410,430,480,520,590,660,850,1000,1000
+,Aggregated,47348,0,350,344.40586719608007,5,1034,97.76577680155445,993.0375320581431,0.0,350,390,410,430,480,520,590,660,850,1000,1000

bench/results/v1/nopooling-nopipeline/requests_1702930179.0353742.csv

@@ -0,0 +1,3 @@
+Type,Name,Request Count,Failure Count,Median Response Time,Average Response Time,Min Response Time,Max Response Time,Average Content Size,Requests/s,Failures/s,50%,66%,75%,80%,90%,95%,98%,99%,99.9%,99.99%,100%
+POST,http://localhost:9001/write,47348,0,350,344.40586719608007,5,1034,97.76577680155445,993.0375320581431,0.0,350,390,410,430,480,520,590,660,850,1000,1000
+,Aggregated,47348,0,350,344.40586719608007,5,1034,97.76577680155445,993.0375320581431,0.0,350,390,410,430,480,520,590,660,850,1000,1000

bench/results/v1/pipeline/100-cmd-per-request/exceptions_1702823234.0957885.csv

@@ -0,0 +1 @@
+Count,Message,Traceback,Nodes

bench/results/v1/pipeline/100-cmd-per-request/failures_1702823230.782065.csv

@@ -0,0 +1 @@
+Method,Name,Error,Occurrences

bench/results/v1/pipeline/100-cmd-per-request/requests_1702823224.1687477.csv

@@ -0,0 +1,3 @@
+Type,Name,Request Count,Failure Count,Median Response Time,Average Response Time,Min Response Time,Max Response Time,Average Content Size,Requests/s,Failures/s,50%,66%,75%,80%,90%,95%,98%,99%,99.9%,99.99%,100%
+POST,http://localhost:9001/pipeline/write,10968,0,620,761.5634573304158,7,1293,9999.697757111597,161.3685696386145,0.0,620,1100,1100,1100,1200,1200,1200,1200,1300,1300,1300
+,Aggregated,10968,0,620,761.5634573304158,7,1293,9999.697757111597,161.3685696386145,0.0,620,1100,1100,1100,1200,1200,1200,1200,1300,1300,1300

bench/results/v1/pipeline/1000-cmd-per-request/README.md

@@ -0,0 +1,3 @@
+Design:
+
+Actix Web and reqwests with request pipelining 

bench/results/v1/pipeline/1000-cmd-per-request/exceptions_1702822848.3234775.csv

@@ -0,0 +1 @@
+Count,Message,Traceback,Nodes

bench/results/v1/pipeline/1000-cmd-per-request/failures_1702822844.312193.csv

@@ -0,0 +1 @@
+Method,Name,Error,Occurrences

bench/results/v1/pipeline/1000-cmd-per-request/requests_1702822817.6318111.csv

@@ -0,0 +1,3 @@
+Type,Name,Request Count,Failure Count,Median Response Time,Average Response Time,Min Response Time,Max Response Time,Average Content Size,Requests/s,Failures/s,50%,66%,75%,80%,90%,95%,98%,99%,99.9%,99.99%,100%
+POST,http://localhost:9001/pipeline/write,1141,0,720,681.7598597721297,69,858,100027.22261174409,16.07784382812381,0.0,720,740,750,760,780,790,810,820,840,860,860
+,Aggregated,1141,0,720,681.7598597721297,69,858,100027.22261174409,16.07784382812381,0.0,720,740,750,760,780,790,810,820,840,860,860

src/lib.rs

@@ -160,6 +160,7 @@ pub async fn start_example_raft_node(
             .service(api::read)
             .service(api::pipeline_read)
             .service(api::pipeline_write)
+            .service(api::ws_write)
         //            .service(api::consistent_read)
     });
 

src/network/api.rs

@@ -15,6 +15,11 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
+use std::sync::{Arc, RwLock};
+
+use actix::dev::channel::channel;
+use actix::WrapFuture;
+
 use actix_web::get;
 use actix_web::post;
 use actix_web::web;
@@ -28,6 +33,7 @@ use web::Json;
 
 use crate::app::DcacheApp;
 use crate::store::DcacheRequest;
+use crate::store::DcacheResponse;
 
 #[post("/write")]
 pub async fn write(
@@ -155,3 +161,86 @@ pub async fn pipeline_write(
     }
     Ok(Json(responses))
 }
+
+use actix::prelude::*;
+use actix::{Actor, StreamHandler};
+use actix_web::{Error, HttpRequest, HttpResponse};
+use actix_web_actors::ws;
+
+/// Define HTTP actor
+struct MyWs {
+    app: Data<DcacheApp>,
+}
+
+impl Actor for MyWs {
+    type Context = ws::WebsocketContext<Self>;
+}
+
+/// Handler for ws::Message message
+impl StreamHandler<Result<ws::Message, ws::ProtocolError>> for MyWs {
+    fn handle(&mut self, msg: Result<ws::Message, ws::ProtocolError>, ctx: &mut Self::Context) {
+        match msg {
+            Ok(ws::Message::Ping(msg)) => ctx.pong(&msg),
+            Ok(ws::Message::Text(text)) => {
+                let msg: super::raft::RaftMessage = serde_json::from_str(&text).unwrap();
+
+                let res: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
+
+                let res2 = res.clone();
+                let app = self.app.clone();
+
+                let fut = async move {
+                    let mut w = res2.write().unwrap();
+
+                    match msg {
+                        super::raft::RaftMessage::VoteRequest(msg) => {
+                            let res = app.raft.vote(msg).await;
+                            panic!("setting vote req");
+                            *w = Some(serde_json::to_string(&res).unwrap());
+                        }
+                        super::raft::RaftMessage::Append(msg) => {
+                            let res = app.raft.append_entries(msg).await;
+                            panic!("setting app req");
+                            *w = Some(serde_json::to_string(&res).unwrap());
+                        }
+                        super::raft::RaftMessage::Snapshot(msg) => {
+                            panic!("setting snap req");
+                            let res = app.raft.install_snapshot(msg).await;
+                            *w = Some(serde_json::to_string(&res).unwrap());
+                        }
+                    }
+                }
+                .into_actor(self);
+                ctx.wait(fut);
+                //loop {
+                println!("running read loop");
+                let res = res.read().unwrap();
+                //       if res.is_some() {
+                let res: &str = res.as_ref().unwrap();
+                ctx.text(res);
+                //       break;
+                //      }
+                //}
+
+                //                if let Ok(res) = res.read() {
+                //                    if res.is_some() {
+                //                        let res: &str = res.as_ref().unwrap();
+                //                        ctx.text(res);
+                //                    }
+                //                };
+            }
+            _ => (),
+        }
+    }
+}
+
+#[get("/ws/write")]
+async fn ws_write(
+    req: HttpRequest,
+    app: Data<DcacheApp>,
+    stream: web::Payload,
+) -> Result<HttpResponse, Error> {
+    let resp = ws::start(MyWs { app }, &req, stream);
+    println!("{:?}", resp);
+    resp
+}

src/network/raft.rs

@@ -19,9 +19,12 @@ use actix_web::post;
 use actix_web::web;
 use actix_web::web::Data;
 use actix_web::Responder;
+use openraft::error::*;
 use openraft::raft::AppendEntriesRequest;
 use openraft::raft::InstallSnapshotRequest;
 use openraft::raft::VoteRequest;
+use openraft::raft::*;
+use serde::*;
 use web::Json;
 
 use crate::app::DcacheApp;
@@ -30,6 +33,25 @@ use crate::DcacheTypeConfig;
 
 // --- Raft communication
 
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum RaftMessage {
+    VoteRequest(VoteRequest<DcacheNodeId>),
+    Append(AppendEntriesRequest<DcacheTypeConfig>),
+    Snapshot(InstallSnapshotRequest<DcacheTypeConfig>),
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum RaftRes {
+    VoteRes(Result<VoteResponse<DcacheNodeId>, RaftError<DcacheNodeId>>),
+    SnapshotRes(
+        Result<
+            InstallSnapshotResponse<DcacheNodeId>,
+            RaftError<DcacheNodeId, InstallSnapshotError>,
+        >,
+    ),
+    AppendRes(Result<AppendEntriesResponse<DcacheNodeId>, RaftError<DcacheNodeId>>),
+}
+
 #[post("/raft-vote")]
 pub async fn vote(
     app: Data<DcacheApp>,

src/network/raft_network_impl.rs

@@ -24,6 +24,12 @@ use std::sync::RwLock;
 use std::time::Duration;
 use std::time::Instant;
 
+use futures_util::{future, pin_mut, StreamExt};
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+use tokio_tungstenite::{connect_async, tungstenite::protocol::Message};
+
+use tokio::sync::mpsc;
+
 use async_trait::async_trait;
 use openraft::error::InstallSnapshotError;
 use openraft::error::NetworkError;
@@ -45,6 +51,9 @@ use serde::Serialize;
 use tokio::sync::mpsc::Sender;
 
 use super::management::HealthStatus;
+use super::raft::{RaftMessage, RaftRes};
+use crate::store::DcacheRequest;
+use crate::store::DcacheResponse;
 use crate::DcacheNodeId;
 use crate::DcacheTypeConfig;
 
@@ -110,10 +119,20 @@ impl RaftNetworkFactory<DcacheTypeConfig> for Arc<DcacheNetwork> {
     type Network = DcacheNetworkConnection;
 
     async fn new_client(&mut self, target: DcacheNodeId, node: &BasicNode) -> Self::Network {
+        let addr = &node.addr;
+        let url = format!("ws://{}/{}", addr, "ws/write");
+
+        let (write, rx) = mpsc::channel(30);
+        let (tx, read) = mpsc::channel(30);
+        let ws_client = WSClient::spawn(rx, tx, url).await;
+
         DcacheNetworkConnection {
             owner: self.clone(),
             target,
             target_node: node.clone(),
+            //            ws_client,
+            read,
+            write,
         }
     }
 }
@@ -122,6 +141,9 @@ pub struct DcacheNetworkConnection {
     owner: Arc<DcacheNetwork>,
     target: DcacheNodeId,
     target_node: BasicNode,
+    //    ws_client: WSClient,
+    write: mpsc::Sender<RaftMessage>,
+    read: mpsc::Receiver<RaftRes>,
 }
 
 #[async_trait]
@@ -133,9 +155,16 @@ impl RaftNetwork<DcacheTypeConfig> for DcacheNetworkConnection {
         AppendEntriesResponse<DcacheNodeId>,
         RPCError<DcacheNodeId, BasicNode, RaftError<DcacheNodeId>>,
     > {
-        self.owner
+        self.write.send(RaftMessage::Append(req)).await.unwrap();
-            .send_rpc(self.target, &self.target_node, "raft-append", req)
+        match self.read.recv().await.unwrap() {
-            .await
+            RaftRes::AppendRes(res) => {
+                res.map_err(|e| RPCError::RemoteError(RemoteError::new(self.target, e)))
+            }
+            _ => unimplemented!(),
+        }
+        //        self.owner
+        //            .send_rpc(self.target, &self.target_node, "raft-append", req)
+        //            .await
     }
 
     async fn send_install_snapshot(
@@ -145,9 +174,16 @@ impl RaftNetwork<DcacheTypeConfig> for DcacheNetworkConnection {
         InstallSnapshotResponse<DcacheNodeId>,
         RPCError<DcacheNodeId, BasicNode, RaftError<DcacheNodeId, InstallSnapshotError>>,
     > {
-        self.owner
+        //        self.owner
-            .send_rpc(self.target, &self.target_node, "raft-snapshot", req)
+        //            .send_rpc(self.target, &self.target_node, "raft-snapshot", req)
-            .await
+        //            .await
+        self.write.send(RaftMessage::Snapshot(req)).await.unwrap();
+        match self.read.recv().await.unwrap() {
+            RaftRes::SnapshotRes(res) => {
+                res.map_err(|e| RPCError::RemoteError(RemoteError::new(self.target, e)))
+            }
+            _ => unimplemented!(),
+        }
     }
 
     async fn send_vote(
@@ -157,8 +193,53 @@ impl RaftNetwork<DcacheTypeConfig> for DcacheNetworkConnection {
         VoteResponse<DcacheNodeId>,
         RPCError<DcacheNodeId, BasicNode, RaftError<DcacheNodeId>>,
     > {
-        self.owner
+        //        self.owner
-            .send_rpc(self.target, &self.target_node, "raft-vote", req)
+        //            .send_rpc(self.target, &self.target_node, "raft-vote", req)
+        //            .await
+        self.write
+            .send(RaftMessage::VoteRequest(req))
             .await
+            .unwrap();
+        match self.read.recv().await.unwrap() {
+            RaftRes::VoteRes(res) => {
+                res.map_err(|e| RPCError::RemoteError(RemoteError::new(self.target, e)))
+            }
+            _ => unimplemented!(),
+        }
+    }
+}
+
+pub struct WSClient;
+
+impl WSClient {
+    pub async fn spawn(
+        mut rx: mpsc::Receiver<RaftMessage>,
+        tx: mpsc::Sender<RaftRes>,
+        url: String,
+    ) {
+        use futures_util::SinkExt;
+
+        let (ws_stream, _) = connect_async(url).await.expect("Failed to connect");
+        println!("WebSocket handshake has been successfully completed");
+
+        let (mut write, mut read) = ws_stream.split();
+
+        let fut = async move {
+            while let Some(msg) = rx.recv().await {
+                write
+                    .send(Message::Text(serde_json::to_string(&msg).unwrap()))
+                    .await
+                    .unwrap();
+
+                match read.next().await.unwrap().unwrap() {
+                    Message::Text(msg) => {
+                        tx.send(serde_json::from_str(&msg).unwrap()).await;
+                    }
+                    _ => (),
+                }
+            }
+        };
+
+        tokio::spawn(fut);
     }
 }

src/store/mod.rs

@@ -23,6 +23,7 @@ use std::ops::RangeBounds;
 use std::sync::Arc;
 use std::sync::Mutex;
 
+use actix::prelude::*;
 use libmcaptcha::cache::messages::CachedPoWConfig;
 use libmcaptcha::AddVisitorResult;
 use libmcaptcha::MCaptcha;
@@ -65,7 +66,8 @@ use libmcaptcha::{master::embedded::master::Master as EmbeddedMaster, system::Sy
 
 pub mod system;
 
-#[derive(Serialize, Deserialize, Clone, Debug)]
+#[derive(Serialize, Deserialize, Clone, Debug, Message)]
+#[rtype(result = "DcacheResponse")]
 pub enum DcacheRequest {
     // master
     AddVisitor(AddVisitor),
@@ -79,7 +81,8 @@ pub enum DcacheRequest {
     DeleteCaptchaResult(DeleteCaptchaResult),
 }
 
-#[derive(Serialize, Deserialize, Debug, Clone)]
+#[derive(Serialize, Deserialize, Message, Debug, Clone)]
+#[rtype(result = "()")]
 pub enum DcacheResponse {
     AddVisitorResult(Option<AddVisitorResult>),
     Empty, // AddCaptcha, RenameCaptcha, RemoveCaptcha, Cachepow, CacheResult,

test.py

@@ -28,6 +28,7 @@ def add_host(host: str, id: int, peer: str):
     params = [id, peer]
     resp = requests.post(f"http://{host}/add-learner", json=params)
     print(f"Adding host {peer}. Status: {resp.status_code}")
+    pprint(resp.json())
 
 
 def switch_to_cluster(host: str, nodes: [int]):
@@ -46,6 +47,7 @@ def write(host, data):
     print(f"RPC Status: {resp.status_code}")
     resp = resp.json()
     if "Err" in resp:
+        pprint(resp)
         leader = resp["Err"]["APIError"]["ForwardToLeader"]["leader_node"]["addr"]
         print(f"Forwarding write to leader {leader}")
         return write(leader, data)
@@ -54,6 +56,7 @@ def write(host, data):
 
 def add_vote(host: str, captcha_id: str):
     resp = write(host, data={"AddVisitor": captcha_id})
+    #pprint(resp.json())
     pprint(resp)
 
 
@@ -79,7 +82,7 @@ def add_captcha(host: str, captcha_id: str):
 
 
 host = "localhost:9001"
-peers = [(2, "localhost:9002"), (3, "localhost:9003"), (4, "localhost:9004")]
+peers = [(2, "localhost:9002"), (3, "localhost:9003")]#, (4, "localhost:9004")]
 captcha_id = "test_1"
 
 def initialize_cluster():
@@ -87,15 +90,25 @@ def initialize_cluster():
     for peer_id, peer in peers:
         add_host(host=host, id=peer_id, peer=peer)
 
-    switch_to_cluster(host, nodes=[1, 2,3,4])
+    #switch_to_cluster(host, nodes=[1, 2,3, 4])
+    switch_to_cluster(host, nodes=[1, 2,3])#, 4])
 
+def add_data():
     add_captcha(host, captcha_id)
     add_vote(host, captcha_id)
-    for _ in range(0, 600):
+#    for _ in range(0, 60):
-        add_vote(host, captcha_id)
+#        add_vote(host, captcha_id)
 
 
 
-if __name__ == "__main__":
+#if __name__ == "__main__":
-
+#    init(host)
-    add_vote("localhost:9002", captcha_id)
+#   # initialize_cluster()
+#    add_data()
+##    switch_to_cluster(host, nodes=[1, 2,3])
+#    add_vote("localhost:9001", captcha_id)
+#    add_vote("localhost:9001", captcha_id)
+##    initialize_cluster()
+##    add_vote("localhost:9002")
+#    for _ in range(0, 60):
+#        add_vote("localhost:9002", captcha_id)