From 44009f6c0d325d765ae94a691065b2023f25d2b7 Mon Sep 17 00:00:00 2001 From: Sagar Arora Date: Fri, 26 Aug 2022 11:08:10 +0530 Subject: [PATCH 1/9] Project-2 Phase 4 Done --- courses/rust/projects/kvs/.idea/.gitignore | 3 + .../projects/kvs/.idea/codeStyles/Project.xml | 7 + .../kvs/.idea/codeStyles/codeStyleConfig.xml | 5 + courses/rust/projects/kvs/.idea/kvs.iml | 13 + courses/rust/projects/kvs/.idea/modules.xml | 8 + courses/rust/projects/kvs/.idea/vcs.xml | 6 + courses/rust/projects/kvs/.vscode/launch.json | 83 +++ courses/rust/projects/kvs/Cargo.lock | 549 ++++++++++++++++++ courses/rust/projects/kvs/Cargo.toml | 18 + courses/rust/projects/kvs/src/lib.rs | 135 +++++ courses/rust/projects/kvs/src/main.rs | 49 ++ courses/rust/projects/kvs/tests/tests.rs | 301 ++++++++++ 12 files changed, 1177 insertions(+) create mode 100644 courses/rust/projects/kvs/.idea/.gitignore create mode 100644 courses/rust/projects/kvs/.idea/codeStyles/Project.xml create mode 100644 courses/rust/projects/kvs/.idea/codeStyles/codeStyleConfig.xml create mode 100644 courses/rust/projects/kvs/.idea/kvs.iml create mode 100644 courses/rust/projects/kvs/.idea/modules.xml create mode 100644 courses/rust/projects/kvs/.idea/vcs.xml create mode 100644 courses/rust/projects/kvs/.vscode/launch.json create mode 100644 courses/rust/projects/kvs/Cargo.lock create mode 100644 courses/rust/projects/kvs/Cargo.toml create mode 100644 courses/rust/projects/kvs/src/lib.rs create mode 100644 courses/rust/projects/kvs/src/main.rs create mode 100644 courses/rust/projects/kvs/tests/tests.rs diff --git a/courses/rust/projects/kvs/.idea/.gitignore b/courses/rust/projects/kvs/.idea/.gitignore new file mode 100644 index 000000000..26d33521a --- /dev/null +++ b/courses/rust/projects/kvs/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/courses/rust/projects/kvs/.idea/codeStyles/Project.xml b/courses/rust/projects/kvs/.idea/codeStyles/Project.xml new file mode 100644 index 000000000..919ce1f1f --- /dev/null +++ b/courses/rust/projects/kvs/.idea/codeStyles/Project.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/courses/rust/projects/kvs/.idea/codeStyles/codeStyleConfig.xml b/courses/rust/projects/kvs/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 000000000..a55e7a179 --- /dev/null +++ b/courses/rust/projects/kvs/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/courses/rust/projects/kvs/.idea/kvs.iml b/courses/rust/projects/kvs/.idea/kvs.iml new file mode 100644 index 000000000..6b5fada07 --- /dev/null +++ b/courses/rust/projects/kvs/.idea/kvs.iml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/courses/rust/projects/kvs/.idea/modules.xml b/courses/rust/projects/kvs/.idea/modules.xml new file mode 100644 index 000000000..ef2379344 --- /dev/null +++ b/courses/rust/projects/kvs/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/courses/rust/projects/kvs/.idea/vcs.xml b/courses/rust/projects/kvs/.idea/vcs.xml new file mode 100644 index 000000000..4fce1d86b --- /dev/null +++ b/courses/rust/projects/kvs/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/courses/rust/projects/kvs/.vscode/launch.json b/courses/rust/projects/kvs/.vscode/launch.json new file mode 100644 index 000000000..fcd6c10de --- /dev/null +++ b/courses/rust/projects/kvs/.vscode/launch.json @@ -0,0 +1,83 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in library 'kvs'", + "cargo": { + "args": [ + "test", + "--no-run", + "--lib", + "--package=kvs" + ], + "filter": { + "name": "kvs", + "kind": "lib" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug executable 'kvs'", + "cargo": { + "args": [ + "build", + "--bin=kvs", + "--package=kvs" + ], + "filter": { + "name": "kvs", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in executable 'kvs'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bin=kvs", + "--package=kvs" + ], + "filter": { + "name": "kvs", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug integration test 'tests'", + "cargo": { + "args": [ + "test", + "--no-run", + "--test=tests", + "--package=kvs" + ], + "filter": { + "name": "tests", + "kind": "test" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + } + ] +} \ No newline at end of file diff --git a/courses/rust/projects/kvs/Cargo.lock b/courses/rust/projects/kvs/Cargo.lock new file mode 100644 index 000000000..16e77b2c0 --- /dev/null +++ b/courses/rust/projects/kvs/Cargo.lock @@ -0,0 +1,549 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "assert_cmd" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc477793bd82ec39799b6f6b3df64938532fdf2ab0d49ef817eac65856a5a1e" +dependencies = [ + "escargot", + "predicates", + "predicates-core", + "predicates-tree", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "3.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54635806b078b7925d6e36810b1755f2a4b5b4d57560432c1ecf60bcbe10602b" +dependencies = [ + "atty", + "bitflags", + "clap_lex", + "indexmap", + "strsim", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "difference" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" + +[[package]] +name = "escargot" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ceb9adbf9874d5d028b5e4c5739d22b71988252b25c9c98fe7cf9738bee84597" +dependencies = [ + "lazy_static", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "fastrand" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +dependencies = [ + "instant", +] + +[[package]] +name = "float-cmp" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1267f4ac4f343772758f7b1bdcbe767c218bbab93bb432acbf5162bbf85a6c4" +dependencies = [ + "num-traits", +] + +[[package]] +name = "gimli" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "indexmap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "itoa" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" + +[[package]] +name = "kvs" +version = "0.1.0" +dependencies = [ + "assert_cmd", + "clap", + "failure", + "predicates", + "serde", + "serde_json", + "tempfile", + "walkdir", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "miniz_oxide" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" +dependencies = [ + "adler", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" +dependencies = [ + "memchr", +] + +[[package]] +name = "os_str_bytes" +version = "6.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "648001efe5d5c0102d8cea768e348da85d90af8ba91f0bea908f157951493cd4" + +[[package]] +name = "predicates" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49cfaf7fdaa3bfacc6fa3e7054e65148878354a5cfddcf661df4c851f8021df" +dependencies = [ + "difference", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da1c2388b1513e1b605fcec39a95e0a9e8ef088f71443ef37099fa9ae6673fcb" + +[[package]] +name = "predicates-tree" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d86de6de25020a36c6d3643a86d9a6a9f552107c0559c60ea03551b5e16c032" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "proc-macro2" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd96a1e8ed2596c337f8eae5f24924ec83f5ad5ab21ea8e455d3566c69fbcaf7" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" + +[[package]] +name = "ryu" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc855a42c7967b7c369eb5860f7164ef1f6f81c20c7cc1141f2a604e18723b03" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f2122636b9fe3b81f1cb25099fcf2d3f542cdb1d45940d56c713158884a05da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "termtree" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b" + +[[package]] +name = "textwrap" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" + +[[package]] +name = "unicode-ident" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15c61ba63f9235225a22310255a29b806b907c9b8c964bcbd0a2c70f3f2deea7" + +[[package]] +name = "unicode-xid" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" + +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/courses/rust/projects/kvs/Cargo.toml b/courses/rust/projects/kvs/Cargo.toml new file mode 100644 index 000000000..57c525e8d --- /dev/null +++ b/courses/rust/projects/kvs/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "kvs" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dev-dependencies] +assert_cmd = "0.11.0" +predicates = "1.0.0" +tempfile = "3.0.7" +walkdir = "2.2.7" + +[dependencies] +clap = "3.2.14" +failure = "0.1.8" +serde_json = "1.0" +serde = { version = "1.0", features = ["derive"] } diff --git a/courses/rust/projects/kvs/src/lib.rs b/courses/rust/projects/kvs/src/lib.rs new file mode 100644 index 000000000..f3d613345 --- /dev/null +++ b/courses/rust/projects/kvs/src/lib.rs @@ -0,0 +1,135 @@ +extern crate core; + +use std::collections::HashMap; +use std::path::PathBuf; +use failure::Fail; +use serde::{Serialize, Deserialize}; +use std::fs::{File, OpenOptions}; +use std::io; +use std::io::{BufReader, BufWriter}; +use std::io::prelude::*; +use crate::KvsCommand::{Rm, Set}; +use crate::KvsError::KeyNotFound; + +pub type Result = std::result::Result; + +#[derive(Fail, Debug)] +#[fail(display = "Error in KVS")] +pub enum KvsError { + #[fail(display = "{}", _0)] + IO(#[cause] io::Error), + #[fail(display = "Error Serializing or Deserializing {}", _0)] + SerdeError(serde_json::Error), + #[fail(display = "Key not found")] + KeyNotFound +} + +impl From for KvsError { + fn from(err: io::Error) -> Self { + return KvsError::IO(err) + } +} + +impl From for KvsError { + fn from(err: serde_json::Error) -> Self { + return KvsError::SerdeError(err) + } +} + +#[derive(Serialize, Deserialize)] +pub enum KvsCommand { + Set(String, String), + Rm(String) +} + +impl KvsCommand { + + pub fn set(key : String, value : String) -> KvsCommand { + Set(key, value) + } + + pub fn rm(key : String) -> KvsCommand { + Rm(key) + } +} +pub struct KvStore { + file : File, + map : HashMap +} + +impl KvStore { + + fn get_file_name() -> &'static str { + return "kvs.bin"; + } + + fn read_from_file(file_path : &mut PathBuf) -> Result> { + let mut map = HashMap::new(); + let file_exists = file_path.as_path().exists(); + if !file_exists { + Ok(map) + } else { + let file = OpenOptions::new() + .read(true) + .open(file_path)?; + + let mut buf_reader = BufReader::new(file); + + serde_json::Deserializer::from_reader(&mut buf_reader) + .into_iter::() + .filter_map(|it| it.ok()) + .for_each(|it| { + match it { + Set(key, val) => map.insert(key, val), + Rm(key) => map.remove(&key), + }; + }); + Ok(map) + } + } + + pub fn open(path: impl Into) -> Result { + let mut file_path = path.into(); + file_path.push(Self::get_file_name()); + let map = Self::read_from_file(&mut file_path)?; + let file = OpenOptions::new() + .create(true) + .append(true) + .open(file_path)?; + + Ok(KvStore{ + file, + map + }) + } + + pub fn set(&mut self, key: String, value: String) -> Result<()> { + let command = KvsCommand::set(key.clone(), value.clone()); + let mut writer = BufWriter::new(&self.file); + serde_json::to_writer(&mut writer, &command)?; + writer.flush()?; + self.map.insert(key, value); + Ok(()) + } + + pub fn get(&self, key: String) -> Result> { + let option = self.map.get(&key); + match option { + Some(x) => Ok(Some(x.clone())), + None => Ok(None), + } + } + + pub fn remove(&mut self, key: String) -> Result<()> { + if self.map.contains_key(&key) { + let command = KvsCommand::rm(key.clone()); + let mut writer = BufWriter::new(&self.file); + serde_json::to_writer(&mut writer, &command)?; + writer.flush()?; + self.map.remove(&key); + Ok(()) + } else { + Err(KeyNotFound) + } + } +} diff --git a/courses/rust/projects/kvs/src/main.rs b/courses/rust/projects/kvs/src/main.rs new file mode 100644 index 000000000..b76a38700 --- /dev/null +++ b/courses/rust/projects/kvs/src/main.rs @@ -0,0 +1,49 @@ +use std::env::current_dir; +use clap::{arg, Command}; +use kvs::{KvStore, Result}; + +fn main() -> Result<()> { + let version = env!("CARGO_PKG_VERSION"); + let matches = Command::new("Kvs") + .version(version) + .subcommands(vec![ + Command::new("set").args(vec![ + arg!([KEY]).required(true), + arg!([VALUE]).required(true), + ]), + Command::new("get").args(vec![arg!([KEY]).required(true)]), + Command::new("rm").args(vec![arg!([KEY]).required(true)]), + ]) + .get_matches(); + + let mut kv_store = KvStore::open(current_dir().unwrap())?; + + match matches.subcommand() { + Some(("set", args))=> { + let key = args.value_of("KEY").unwrap(); + let value = args.value_of("VALUE").unwrap(); + kv_store.set(key.to_string(), value.clone().to_string())?; + }, + Some(("get", args)) => { + let key = args.value_of("KEY").unwrap(); + let value = kv_store.get(key.to_string())?; + match value { + Some(x) => println!("{}", x), + None => println!("Key not found"), + } + }, + Some(("rm", args)) => { + let key = args.value_of("KEY").unwrap(); + let result = kv_store.remove(key.to_string()); + match result { + Err(err) => { + println!("{}", err); + std::process::exit(1); + }, + _ => {}, + } + }, + _ => std::process::exit(1), + } + Ok(()) +} diff --git a/courses/rust/projects/kvs/tests/tests.rs b/courses/rust/projects/kvs/tests/tests.rs new file mode 100644 index 000000000..95b34db1a --- /dev/null +++ b/courses/rust/projects/kvs/tests/tests.rs @@ -0,0 +1,301 @@ +use assert_cmd::prelude::*; +use kvs::{KvStore, Result}; +use predicates::ord::eq; +use predicates::str::{contains, is_empty, PredicateStrExt}; +use std::process::Command; +use tempfile::TempDir; +use walkdir::WalkDir; + +// `kvs` with no args should exit with a non-zero code. +#[test] +fn cli_no_args() { + Command::cargo_bin("kvs").unwrap().assert().failure(); +} + +// `kvs -V` should print the version +#[test] +fn cli_version() { + Command::cargo_bin("kvs") + .unwrap() + .args(&["-V"]) + .assert() + .stdout(contains(env!("CARGO_PKG_VERSION"))); +} + +// `kvs get ` should print "Key not found" for a non-existent key and exit with zero. +#[test] +fn cli_get_non_existent_key() { + let temp_dir = TempDir::new().unwrap(); + Command::cargo_bin("kvs") + .unwrap() + .args(&["get", "key1"]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout(eq("Key not found").trim()); +} + +// `kvs rm ` should print "Key not found" for an empty database and exit with non-zero code. +#[test] +fn cli_rm_non_existent_key() { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + Command::cargo_bin("kvs") + .unwrap() + .args(&["rm", "key1"]) + .current_dir(&temp_dir) + .assert() + .failure() + .stdout(eq("Key not found").trim()); +} + +// `kvs set ` should print nothing and exit with zero. +#[test] +fn cli_set() { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + Command::cargo_bin("kvs") + .unwrap() + .args(&["set", "key1", "value1"]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout(is_empty()); +} + +#[test] +fn cli_get_stored() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + + let mut store = KvStore::open(temp_dir.path())?; + store.set("key1".to_owned(), "value1".to_owned())?; + store.set("key2".to_owned(), "value2".to_owned())?; + drop(store); + + Command::cargo_bin("kvs") + .unwrap() + .args(&["get", "key1"]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout(eq("value1").trim()); + + Command::cargo_bin("kvs") + .unwrap() + .args(&["get", "key2"]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout(eq("value2").trim()); + + Ok(()) +} + +// `kvs rm ` should print nothing and exit with zero. +#[test] +fn cli_rm_stored() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + + let mut store = KvStore::open(temp_dir.path())?; + store.set("key1".to_owned(), "value1".to_owned())?; + drop(store); + + Command::cargo_bin("kvs") + .unwrap() + .args(&["rm", "key1"]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout(is_empty()); + + Command::cargo_bin("kvs") + .unwrap() + .args(&["get", "key1"]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout(eq("Key not found").trim()); + + Ok(()) +} + +#[test] +fn cli_invalid_get() { + Command::cargo_bin("kvs") + .unwrap() + .args(&["get"]) + .assert() + .failure(); + + Command::cargo_bin("kvs") + .unwrap() + .args(&["get", "extra", "field"]) + .assert() + .failure(); +} + +#[test] +fn cli_invalid_set() { + Command::cargo_bin("kvs") + .unwrap() + .args(&["set"]) + .assert() + .failure(); + + Command::cargo_bin("kvs") + .unwrap() + .args(&["set", "missing_field"]) + .assert() + .failure(); + + Command::cargo_bin("kvs") + .unwrap() + .args(&["set", "extra", "extra", "field"]) + .assert() + .failure(); +} + +#[test] +fn cli_invalid_rm() { + Command::cargo_bin("kvs") + .unwrap() + .args(&["rm"]) + .assert() + .failure(); + + Command::cargo_bin("kvs") + .unwrap() + .args(&["rm", "extra", "field"]) + .assert() + .failure(); +} + +#[test] +fn cli_invalid_subcommand() { + Command::cargo_bin("kvs") + .unwrap() + .args(&["unknown", "subcommand"]) + .assert() + .failure(); +} + +// Should get previously stored value. +#[test] +fn get_stored_value() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut store = KvStore::open(temp_dir.path())?; + + store.set("key1".to_owned(), "value1".to_owned())?; + store.set("key2".to_owned(), "value2".to_owned())?; + + assert_eq!(store.get("key1".to_owned())?, Some("value1".to_owned())); + assert_eq!(store.get("key2".to_owned())?, Some("value2".to_owned())); + + // Open from disk again and check persistent data. + drop(store); + let mut store = KvStore::open(temp_dir.path())?; + assert_eq!(store.get("key1".to_owned())?, Some("value1".to_owned())); + assert_eq!(store.get("key2".to_owned())?, Some("value2".to_owned())); + + Ok(()) +} + +// Should overwrite existent value. +#[test] +fn overwrite_value() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut store = KvStore::open(temp_dir.path())?; + + store.set("key1".to_owned(), "value1".to_owned())?; + assert_eq!(store.get("key1".to_owned())?, Some("value1".to_owned())); + store.set("key1".to_owned(), "value2".to_owned())?; + assert_eq!(store.get("key1".to_owned())?, Some("value2".to_owned())); + + // Open from disk again and check persistent data. + drop(store); + let mut store = KvStore::open(temp_dir.path())?; + assert_eq!(store.get("key1".to_owned())?, Some("value2".to_owned())); + store.set("key1".to_owned(), "value3".to_owned())?; + assert_eq!(store.get("key1".to_owned())?, Some("value3".to_owned())); + + Ok(()) +} + +// Should get `None` when getting a non-existent key. +#[test] +fn get_non_existent_value() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut store = KvStore::open(temp_dir.path())?; + + store.set("key1".to_owned(), "value1".to_owned())?; + assert_eq!(store.get("key2".to_owned())?, None); + + // Open from disk again and check persistent data. + drop(store); + let mut store = KvStore::open(temp_dir.path())?; + assert_eq!(store.get("key2".to_owned())?, None); + + Ok(()) +} + +#[test] +fn remove_non_existent_key() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut store = KvStore::open(temp_dir.path())?; + assert!(store.remove("key1".to_owned()).is_err()); + Ok(()) +} + +#[test] +fn remove_key() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut store = KvStore::open(temp_dir.path())?; + store.set("key1".to_owned(), "value1".to_owned())?; + assert!(store.remove("key1".to_owned()).is_ok()); + assert_eq!(store.get("key1".to_owned())?, None); + Ok(()) +} + +// Insert data until total size of the directory decreases. +// Test data correctness after compaction. +#[test] +fn compaction() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut store = KvStore::open(temp_dir.path())?; + + let dir_size = || { + let entries = WalkDir::new(temp_dir.path()).into_iter(); + let len: walkdir::Result = entries + .map(|res| { + res.and_then(|entry| entry.metadata()) + .map(|metadata| metadata.len()) + }) + .sum(); + len.expect("fail to get directory size") + }; + + let mut current_size = dir_size(); + for iter in 0..1000 { + for key_id in 0..1000 { + let key = format!("key{}", key_id); + let value = format!("{}", iter); + store.set(key, value)?; + } + + let new_size = dir_size(); + if new_size > current_size { + current_size = new_size; + continue; + } + // Compaction triggered. + + drop(store); + // reopen and check content. + let mut store = KvStore::open(temp_dir.path())?; + for key_id in 0..1000 { + let key = format!("key{}", key_id); + assert_eq!(store.get(key)?, Some(format!("{}", iter))); + } + return Ok(()); + } + + panic!("No compaction detected"); +} From 0af4f273e456a9ea65dc644957d358842c08ffff Mon Sep 17 00:00:00 2001 From: Sagar Arora Date: Fri, 26 Aug 2022 12:27:40 +0530 Subject: [PATCH 2/9] Project-2 Part-5 Done --- courses/rust/projects/kvs/src/lib.rs | 84 +++++++++++++++++++--------- 1 file changed, 58 insertions(+), 26 deletions(-) diff --git a/courses/rust/projects/kvs/src/lib.rs b/courses/rust/projects/kvs/src/lib.rs index f3d613345..2f0d5b361 100644 --- a/courses/rust/projects/kvs/src/lib.rs +++ b/courses/rust/projects/kvs/src/lib.rs @@ -6,6 +6,7 @@ use failure::Fail; use serde::{Serialize, Deserialize}; use std::fs::{File, OpenOptions}; use std::io; +use std::io::SeekFrom; use std::io::{BufReader, BufWriter}; use std::io::prelude::*; use crate::KvsCommand::{Rm, Set}; @@ -53,8 +54,10 @@ impl KvsCommand { } } pub struct KvStore { - file : File, - map : HashMap + buf_reader : BufReader, + buf_writer : BufWriter, + map : HashMap, + cur_offset : u64 } impl KvStore { @@ -63,7 +66,7 @@ impl KvStore { return "kvs.bin"; } - fn read_from_file(file_path : &mut PathBuf) -> Result> { + fn read_from_file(file_path : &mut PathBuf) -> Result> { let mut map = HashMap::new(); let file_exists = file_path.as_path().exists(); if !file_exists { @@ -74,16 +77,24 @@ impl KvStore { .open(file_path)?; let mut buf_reader = BufReader::new(file); - - serde_json::Deserializer::from_reader(&mut buf_reader) - .into_iter::() - .filter_map(|it| it.ok()) - .for_each(|it| { - match it { - Set(key, val) => map.insert(key, val), - Rm(key) => map.remove(&key), - }; - }); + let mut itr = serde_json::Deserializer::from_reader(&mut buf_reader) + .into_iter::(); + + let mut offset = 0; + loop { + let kvs_command_option = itr.next(); + match kvs_command_option { + Some(kvs_command_result) => { + let kvs_command = kvs_command_result.unwrap(); + match kvs_command { + Set(key, _) => map.insert(key, offset), + Rm(key) => map.remove(&key), + }; + }, + None => break + } + offset += itr.byte_offset() as u64; + } Ok(map) } } @@ -92,30 +103,50 @@ impl KvStore { let mut file_path = path.into(); file_path.push(Self::get_file_name()); let map = Self::read_from_file(&mut file_path)?; - let file = OpenOptions::new() + + let file_2 = OpenOptions::new() .create(true) .append(true) - .open(file_path)?; + .open(&file_path)?; + let offset = file_2.metadata()?.len(); + let buf_writer = BufWriter::new(file_2); + + let file_1 = OpenOptions::new() + .read(true) + .open(&file_path)?; + + let buf_reader = BufReader::new(file_1); Ok(KvStore{ - file, - map + buf_reader, + buf_writer, + map, + cur_offset : offset }) } pub fn set(&mut self, key: String, value: String) -> Result<()> { let command = KvsCommand::set(key.clone(), value.clone()); - let mut writer = BufWriter::new(&self.file); - serde_json::to_writer(&mut writer, &command)?; - writer.flush()?; - self.map.insert(key, value); + let json_str = serde_json::to_string(&command)?; + let bytes_written = self.buf_writer.write(json_str.as_bytes())?; + self.buf_writer.flush()?; + self.map.insert(key,self.cur_offset); + self.cur_offset += bytes_written as u64; Ok(()) } - pub fn get(&self, key: String) -> Result> { + pub fn get(&mut self, key: String) -> Result> { let option = self.map.get(&key); match option { - Some(x) => Ok(Some(x.clone())), + Some(offset) => { + self.buf_reader.seek(SeekFrom::Start(offset.clone()))?; + let kvs_command = serde_json::Deserializer::from_reader(&mut self.buf_reader) + .into_iter::().next().unwrap()?; + match kvs_command { + Set(_, v) => Ok(Some(v)), + _ => panic!("No Set command found") + } + } None => Ok(None), } } @@ -123,9 +154,10 @@ impl KvStore { pub fn remove(&mut self, key: String) -> Result<()> { if self.map.contains_key(&key) { let command = KvsCommand::rm(key.clone()); - let mut writer = BufWriter::new(&self.file); - serde_json::to_writer(&mut writer, &command)?; - writer.flush()?; + let json_str = serde_json::to_string(&command)?; + let bytes_written = self.buf_writer.write(json_str.as_bytes())?; + self.buf_writer.flush()?; + self.cur_offset += bytes_written as u64; self.map.remove(&key); Ok(()) } else { From 15e892a5378f1ec08efeb27739a3248bb7b8b911 Mon Sep 17 00:00:00 2001 From: Sagar Arora Date: Mon, 10 Oct 2022 10:47:49 +0530 Subject: [PATCH 3/9] Except Compaction Done --- courses/rust/projects/kvs/Cargo.lock | 54 +++++ courses/rust/projects/kvs/Cargo.toml | 1 + courses/rust/projects/kvs/src/lib.rs | 283 ++++++++++++++++++++------- 3 files changed, 266 insertions(+), 72 deletions(-) diff --git a/courses/rust/projects/kvs/Cargo.lock b/courses/rust/projects/kvs/Cargo.lock index 16e77b2c0..482ce1ad9 100644 --- a/courses/rust/projects/kvs/Cargo.lock +++ b/courses/rust/projects/kvs/Cargo.lock @@ -170,6 +170,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "getrandom" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "gimli" version = "0.26.2" @@ -224,6 +235,7 @@ dependencies = [ "clap", "failure", "predicates", + "rand", "serde", "serde_json", "tempfile", @@ -296,6 +308,12 @@ version = "6.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "648001efe5d5c0102d8cea768e348da85d90af8ba91f0bea908f157951493cd4" +[[package]] +name = "ppv-lite86" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" + [[package]] name = "predicates" version = "1.0.8" @@ -343,6 +361,36 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -517,6 +565,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "winapi" version = "0.3.9" diff --git a/courses/rust/projects/kvs/Cargo.toml b/courses/rust/projects/kvs/Cargo.toml index 57c525e8d..096d740a8 100644 --- a/courses/rust/projects/kvs/Cargo.toml +++ b/courses/rust/projects/kvs/Cargo.toml @@ -16,3 +16,4 @@ clap = "3.2.14" failure = "0.1.8" serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } +rand = "0.8.5" diff --git a/courses/rust/projects/kvs/src/lib.rs b/courses/rust/projects/kvs/src/lib.rs index 2f0d5b361..4eaf5ed7f 100644 --- a/courses/rust/projects/kvs/src/lib.rs +++ b/courses/rust/projects/kvs/src/lib.rs @@ -1,14 +1,17 @@ extern crate core; +use rand::Rng; use std::collections::HashMap; use std::path::PathBuf; +use std::time::{SystemTime, UNIX_EPOCH}; use failure::Fail; use serde::{Serialize, Deserialize}; use std::fs::{File, OpenOptions}; -use std::io; -use std::io::SeekFrom; -use std::io::{BufReader, BufWriter}; +use std::{io}; use std::io::prelude::*; +use std::io::{SeekFrom, Seek}; +use std::io::{BufReader, BufWriter}; + use crate::KvsCommand::{Rm, Set}; use crate::KvsError::KeyNotFound; @@ -39,111 +42,245 @@ impl From for KvsError { #[derive(Serialize, Deserialize)] pub enum KvsCommand { - Set(String, String), - Rm(String) + Set(String, String, u128), + Rm(String, u128) } impl KvsCommand { - pub fn set(key : String, value : String) -> KvsCommand { - Set(key, value) + pub fn set(key : String, value : String, timestamp : u128) -> KvsCommand { + Set(key, value, timestamp) } - pub fn rm(key : String) -> KvsCommand { - Rm(key) + pub fn rm(key : String, timestamp : u128) -> KvsCommand { + Rm(key, timestamp) } } pub struct KvStore { + dir_path : PathBuf, + files : HashMap, + map : HashMap, + cur_offset : u64, + cur_index : u32 +} + +struct KvsEntry { + file_index : u32, + timestamp : u128, + offset : u64 +} + +impl KvsEntry { + + fn new(file_index : u32, timestamp : u128, offset : u64) -> Self { + KvsEntry {file_index, timestamp, offset} + } + + fn get_timestamp(&self) -> u128 { + self.timestamp + } + +} + +struct KvsFile { + file_path : PathBuf, buf_reader : BufReader, buf_writer : BufWriter, - map : HashMap, - cur_offset : u64 + index : u32 } impl KvStore { - fn get_file_name() -> &'static str { - return "kvs.bin"; + fn get_timestamp() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap().as_millis() + } + + fn get_file_name() -> String { + let mut rng = rand::thread_rng(); + let n1: u8 = rng.gen(); + format!("kvs{}.bin", n1) } - fn read_from_file(file_path : &mut PathBuf) -> Result> { - let mut map = HashMap::new(); - let file_exists = file_path.as_path().exists(); - if !file_exists { - Ok(map) - } else { - let file = OpenOptions::new() - .read(true) - .open(file_path)?; + fn create_new_file(dir_path : &PathBuf, index : u32) -> KvsFile { + let file_name = Self::get_file_name(); + let mut file_path = dir_path.clone(); + file_path.push(file_name); + Self::create_kvs_file(&file_path, index) + } + + fn find_all_file_in_dir(dir_path : &PathBuf) -> Vec { + let mut files_vec = Vec::new(); + for entry in std::fs::read_dir(dir_path).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + + let metadata = std::fs::metadata(&path).unwrap(); + if metadata.is_file() && path.extension().unwrap() == "bin" { + files_vec.push(path); + } + } + files_vec + } + + fn get_kvs_files(file_paths : Vec) -> Vec { + let mut files = vec![]; + let mut itr = 0; + for file_path in file_paths { + let kvs_file = Self::create_kvs_file(&file_path, itr); + files.push(kvs_file); + itr += 1; + } + files + } + + fn create_kvs_file(file_path : &PathBuf, index : u32) -> KvsFile { + let buf_writer = Self::create_buf_writer(file_path); + let buf_reader = Self::create_buf_reader(file_path); + KvsFile { + file_path : file_path.clone(), + buf_reader, + buf_writer, + index + } + } + + fn create_buf_reader(file_path : &PathBuf) -> BufReader { + let file = OpenOptions::new() + .read(true) + .open(file_path).unwrap(); + + let buf_reader = BufReader::new(file); + buf_reader + } + + fn create_buf_writer(file_path : &PathBuf) -> BufWriter { + let file = OpenOptions::new() + .create(true) + .append(true) + .open(file_path).unwrap(); + let buf_writer = BufWriter::new(file); + buf_writer + } + + fn read_from_file(kvs_file : &mut KvsFile) -> Result<(HashMap, HashMap)> { + let mut set_map = HashMap::new(); + let mut del_map = HashMap::new(); - let mut buf_reader = BufReader::new(file); - let mut itr = serde_json::Deserializer::from_reader(&mut buf_reader) + let mut itr = serde_json::Deserializer::from_reader(&mut kvs_file.buf_reader) .into_iter::(); - let mut offset = 0; - loop { - let kvs_command_option = itr.next(); - match kvs_command_option { - Some(kvs_command_result) => { - let kvs_command = kvs_command_result.unwrap(); - match kvs_command { - Set(key, _) => map.insert(key, offset), - Rm(key) => map.remove(&key), - }; - }, - None => break - } - offset += itr.byte_offset() as u64; + let mut offset = 0; + loop { + let kvs_command_option = itr.next(); + match kvs_command_option { + Some(kvs_command_result) => { + let kvs_command = kvs_command_result.unwrap(); + match kvs_command { + Set(key, _, time_stamp) => { + set_map.insert(key, KvsEntry::new(kvs_file.index, time_stamp, offset)); + }, + Rm(key, time_stamp) => { + del_map.insert(key, KvsEntry::new(kvs_file.index, time_stamp, offset)); + } + }; + }, + None => break } - Ok(map) + offset += itr.byte_offset() as u64; } + kvs_file.buf_reader.seek(SeekFrom::Start(0)).unwrap(); + Ok((set_map, del_map)) } pub fn open(path: impl Into) -> Result { - let mut file_path = path.into(); - file_path.push(Self::get_file_name()); - let map = Self::read_from_file(&mut file_path)?; - - let file_2 = OpenOptions::new() - .create(true) - .append(true) - .open(&file_path)?; - let offset = file_2.metadata()?.len(); - let buf_writer = BufWriter::new(file_2); - - let file_1 = OpenOptions::new() - .read(true) - .open(&file_path)?; + let dir_path = path.into(); + let file_paths = Self::find_all_file_in_dir(&dir_path); + let mut files = Self::get_kvs_files(file_paths); + let mut set_map : HashMap = HashMap::new(); + let mut del_map : HashMap = HashMap::new(); + for mut file in &mut files { + let out = Self::read_from_file(&mut file).unwrap(); + let s_map = out.0; + for s_map_entry in s_map { + if set_map.contains_key(&s_map_entry.0) { + let cur_value = set_map.get(&s_map_entry.0).unwrap(); + if s_map_entry.1.timestamp > cur_value.get_timestamp() { + set_map.insert(s_map_entry.0, s_map_entry.1); + } + } else { + set_map.insert(s_map_entry.0, s_map_entry.1); + } + } - let buf_reader = BufReader::new(file_1); + let d_map = out.1; + for d_map_entry in d_map { + if del_map.contains_key(&d_map_entry.0) { + let cur_value = del_map.get(&d_map_entry.0).unwrap(); + if d_map_entry.1.timestamp > cur_value.get_timestamp() { + del_map.insert(d_map_entry.0, d_map_entry.1); + } + } else { + del_map.insert(d_map_entry.0, d_map_entry.1); + } + } + } - Ok(KvStore{ - buf_reader, - buf_writer, - map, - cur_offset : offset + let mut files_map : HashMap = HashMap::new(); + for file in files { + files_map.insert(file.index, file); + } + + let mut final_map = HashMap::new(); + for entry in set_map { + final_map.insert(entry.0, entry.1); + } + + for entry in del_map { + if final_map.contains_key(&entry.0) { + let cur_val = final_map.get(&entry.0).unwrap(); + if cur_val.get_timestamp() < entry.1.timestamp { + final_map.remove(&entry.0); + } + } + } + + let cur_index = files_map.len() as u32; + let cur_file = Self::create_new_file(&dir_path, cur_index); + files_map.insert(cur_index, cur_file); + + Ok(KvStore { + dir_path, + files : files_map, + map : final_map, + cur_offset : 0, + cur_index }) + } pub fn set(&mut self, key: String, value: String) -> Result<()> { - let command = KvsCommand::set(key.clone(), value.clone()); + let timestamp = Self::get_timestamp(); + let command = KvsCommand::set(key.clone(), value.clone(), timestamp); let json_str = serde_json::to_string(&command)?; - let bytes_written = self.buf_writer.write(json_str.as_bytes())?; - self.buf_writer.flush()?; - self.map.insert(key,self.cur_offset); + let cur_file = self.files.get_mut(&self.cur_index).unwrap(); + let bytes_written = cur_file.buf_writer.write(json_str.as_bytes())?; + cur_file.buf_writer.flush()?; + self.map.insert(key,KvsEntry::new(cur_file.index, timestamp, self.cur_offset)); self.cur_offset += bytes_written as u64; Ok(()) } pub fn get(&mut self, key: String) -> Result> { - let option = self.map.get(&key); - match option { - Some(offset) => { - self.buf_reader.seek(SeekFrom::Start(offset.clone()))?; - let kvs_command = serde_json::Deserializer::from_reader(&mut self.buf_reader) + match self.map.get(&key) { + Some(entry) => { + let kvs_file = self.files.get_mut(&entry.file_index).unwrap(); + kvs_file.buf_reader.seek(SeekFrom::Start(entry.offset.clone()))?; + let kvs_command = serde_json::Deserializer::from_reader(&mut kvs_file.buf_reader) .into_iter::().next().unwrap()?; match kvs_command { - Set(_, v) => Ok(Some(v)), + Set(_, v, _) => Ok(Some(v)), _ => panic!("No Set command found") } } @@ -153,10 +290,12 @@ impl KvStore { pub fn remove(&mut self, key: String) -> Result<()> { if self.map.contains_key(&key) { - let command = KvsCommand::rm(key.clone()); + let timestamp = Self::get_timestamp(); + let command = KvsCommand::rm(key.clone(), timestamp); let json_str = serde_json::to_string(&command)?; - let bytes_written = self.buf_writer.write(json_str.as_bytes())?; - self.buf_writer.flush()?; + let cur_file = self.files.get_mut(&self.cur_index).unwrap(); + let bytes_written = cur_file.buf_writer.write(json_str.as_bytes())?; + cur_file.buf_writer.flush()?; self.cur_offset += bytes_written as u64; self.map.remove(&key); Ok(()) From 4a959d7475f4770a5f710f4911f9c75faf246f3a Mon Sep 17 00:00:00 2001 From: Sagar Arora Date: Sun, 16 Oct 2022 23:09:41 +0530 Subject: [PATCH 4/9] Part 7 done, Project 2 Completed --- courses/rust/projects/kvs/src/lib.rs | 82 ++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 5 deletions(-) diff --git a/courses/rust/projects/kvs/src/lib.rs b/courses/rust/projects/kvs/src/lib.rs index 4eaf5ed7f..1165be503 100644 --- a/courses/rust/projects/kvs/src/lib.rs +++ b/courses/rust/projects/kvs/src/lib.rs @@ -7,7 +7,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use failure::Fail; use serde::{Serialize, Deserialize}; use std::fs::{File, OpenOptions}; -use std::{io}; +use std::io; use std::io::prelude::*; use std::io::{SeekFrom, Seek}; use std::io::{BufReader, BufWriter}; @@ -97,14 +97,14 @@ impl KvStore { .unwrap().as_millis() } - fn get_file_name() -> String { + fn get_file_name(extension : &str) -> String { let mut rng = rand::thread_rng(); let n1: u8 = rng.gen(); - format!("kvs{}.bin", n1) + format!("kvs{}_{}.{}", n1, Self::get_timestamp(), extension) } fn create_new_file(dir_path : &PathBuf, index : u32) -> KvsFile { - let file_name = Self::get_file_name(); + let file_name = Self::get_file_name("bin"); let mut file_path = dir_path.clone(); file_path.push(file_name); Self::create_kvs_file(&file_path, index) @@ -188,7 +188,8 @@ impl KvStore { }, None => break } - offset += itr.byte_offset() as u64; + let itr_offset = itr.byte_offset(); + offset = itr_offset as u64; } kvs_file.buf_reader.seek(SeekFrom::Start(0)).unwrap(); Ok((set_map, del_map)) @@ -269,6 +270,8 @@ impl KvStore { cur_file.buf_writer.flush()?; self.map.insert(key,KvsEntry::new(cur_file.index, timestamp, self.cur_offset)); self.cur_offset += bytes_written as u64; + self.check_cur_file_over_size(); + self.compaction_required(); Ok(()) } @@ -298,9 +301,78 @@ impl KvStore { cur_file.buf_writer.flush()?; self.cur_offset += bytes_written as u64; self.map.remove(&key); + self.check_cur_file_over_size(); + self.compaction_required(); Ok(()) } else { Err(KeyNotFound) } } + + fn check_cur_file_over_size(&mut self) -> () { + if self.cur_offset >= 1_000_000 { + let file_index = self.get_file_index(); + self.files.insert(file_index, Self::create_new_file(&self.dir_path, file_index)); + self.cur_index = file_index; + self.cur_offset = 0; + } + } + + fn compaction_required(&mut self) -> () { + if self.files.len() == 4 { + self.compact().unwrap(); + } + } + + fn get_file_index(&self) -> u32 { + let mut rng = rand::thread_rng(); + loop { + let index = rng.gen::(); + if !self.files.contains_key(&index) { + return index.clone() + } + } + } + + fn compact(&mut self) -> Result<()> { + + let compaction_file_index = self.get_file_index(); + self.files.insert(compaction_file_index, Self::create_new_file(&self.dir_path, compaction_file_index)); + + let keys = self.map.keys().map(|x| x.clone()).collect::>(); + + let mut offset = 0; + for entry in keys { + let value = self.get(entry.clone()).unwrap().unwrap(); + let kvs_entry = self.map.get(&entry).unwrap(); + let command = KvsCommand::set(entry.clone(), value.clone(),kvs_entry.get_timestamp()); + let json_str = serde_json::to_string(&command)?; + let compaction_file = self.files.get_mut(&compaction_file_index).unwrap(); + let bytes_written = compaction_file.buf_writer.write(json_str.as_bytes())?; + compaction_file.buf_writer.flush()?; + self.map.insert(entry.clone(), KvsEntry::new(compaction_file_index, kvs_entry.get_timestamp(), offset)); + offset += bytes_written as u64; + } + + let cur_file_index = self.get_file_index(); + self.files.insert(cur_file_index, Self::create_new_file(&self.dir_path, cur_file_index)); + self.cur_index = cur_file_index; + self.cur_offset = 0; + + let file_to_be_removed_indexs = self.files.keys() + .map(|x| x.clone()) + .into_iter() + .collect::>(); + + for file_index in file_to_be_removed_indexs { + if file_index == compaction_file_index || file_index == cur_file_index { + // DO Nothing + } else { + let file = self.files.remove(&file_index).unwrap(); + std::fs::remove_file(file.file_path).unwrap(); + } + } + + Ok(()) + } } From 43730e78ad1e94d50b74af73dfa6d16f5ffa5dff Mon Sep 17 00:00:00 2001 From: Sagar Arora Date: Sun, 26 Feb 2023 18:36:25 +0530 Subject: [PATCH 5/9] Project3 - Adding 2 binaries --- .../rust/projects/kvs_3/.vscode/launch.json | 83 ++ courses/rust/projects/kvs_3/Cargo.lock | 1092 +++++++++++++++++ courses/rust/projects/kvs_3/Cargo.toml | 21 + courses/rust/projects/kvs_3/README.md | 486 ++++++++ .../rust/projects/kvs_3/src/bin/kvs-client.rs | 49 + .../rust/projects/kvs_3/src/bin/kvs-server.rs | 29 + courses/rust/projects/kvs_3/src/lib.rs | 378 ++++++ courses/rust/projects/kvs_3/tests/cli.rs | 337 +++++ courses/rust/projects/kvs_3/tests/kv_store.rs | 126 ++ 9 files changed, 2601 insertions(+) create mode 100644 courses/rust/projects/kvs_3/.vscode/launch.json create mode 100644 courses/rust/projects/kvs_3/Cargo.lock create mode 100644 courses/rust/projects/kvs_3/Cargo.toml create mode 100644 courses/rust/projects/kvs_3/README.md create mode 100644 courses/rust/projects/kvs_3/src/bin/kvs-client.rs create mode 100644 courses/rust/projects/kvs_3/src/bin/kvs-server.rs create mode 100644 courses/rust/projects/kvs_3/src/lib.rs create mode 100644 courses/rust/projects/kvs_3/tests/cli.rs create mode 100644 courses/rust/projects/kvs_3/tests/kv_store.rs diff --git a/courses/rust/projects/kvs_3/.vscode/launch.json b/courses/rust/projects/kvs_3/.vscode/launch.json new file mode 100644 index 000000000..fcd6c10de --- /dev/null +++ b/courses/rust/projects/kvs_3/.vscode/launch.json @@ -0,0 +1,83 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in library 'kvs'", + "cargo": { + "args": [ + "test", + "--no-run", + "--lib", + "--package=kvs" + ], + "filter": { + "name": "kvs", + "kind": "lib" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug executable 'kvs'", + "cargo": { + "args": [ + "build", + "--bin=kvs", + "--package=kvs" + ], + "filter": { + "name": "kvs", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in executable 'kvs'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bin=kvs", + "--package=kvs" + ], + "filter": { + "name": "kvs", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug integration test 'tests'", + "cargo": { + "args": [ + "test", + "--no-run", + "--test=tests", + "--package=kvs" + ], + "filter": { + "name": "tests", + "kind": "test" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + } + ] +} \ No newline at end of file diff --git a/courses/rust/projects/kvs_3/Cargo.lock b/courses/rust/projects/kvs_3/Cargo.lock new file mode 100644 index 000000000..2e1e63a3e --- /dev/null +++ b/courses/rust/projects/kvs_3/Cargo.lock @@ -0,0 +1,1092 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "assert_cmd" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc477793bd82ec39799b6f6b3df64938532fdf2ab0d49ef817eac65856a5a1e" +dependencies = [ + "escargot", + "predicates", + "predicates-core", + "predicates-tree", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78" +dependencies = [ + "autocfg 1.1.0", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bumpalo" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "bitflags", + "textwrap 0.11.0", + "unicode-width", +] + +[[package]] +name = "clap" +version = "3.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" +dependencies = [ + "atty", + "bitflags", + "clap_lex", + "indexmap", + "strsim", + "termcolor", + "textwrap 0.16.0", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "criterion" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f" +dependencies = [ + "atty", + "cast", + "clap 2.34.0", + "criterion-plot", + "csv", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +dependencies = [ + "autocfg 1.1.0", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "csv" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af91f40b7355f82b0a891f50e70399475945bb0b0da4f1700ce60761c9d3e359" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + +[[package]] +name = "difference" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + +[[package]] +name = "escargot" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ceb9adbf9874d5d028b5e4c5739d22b71988252b25c9c98fe7cf9738bee84597" +dependencies = [ + "lazy_static", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "float-cmp" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1267f4ac4f343772758f7b1bdcbe767c218bbab93bb432acbf5162bbf85a6c4" +dependencies = [ + "num-traits", +] + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "getrandom" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" + +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + +[[package]] +name = "indexmap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +dependencies = [ + "autocfg 1.1.0", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" + +[[package]] +name = "js-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "kvs" +version = "0.1.0" +dependencies = [ + "assert_cmd", + "clap 3.2.23", + "criterion", + "failure", + "predicates", + "rand 0.6.5", + "rand 0.8.5", + "serde", + "serde_json", + "tempfile", + "walkdir", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.139" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg 1.1.0", +] + +[[package]] +name = "miniz_oxide" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +dependencies = [ + "adler", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg 1.1.0", +] + +[[package]] +name = "num_cpus" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +dependencies = [ + "hermit-abi 0.2.6", + "libc", +] + +[[package]] +name = "object" +version = "0.30.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" + +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + +[[package]] +name = "os_str_bytes" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" + +[[package]] +name = "plotters" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" + +[[package]] +name = "plotters-svg" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "predicates" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49cfaf7fdaa3bfacc6fa3e7054e65148878354a5cfddcf661df4c851f8021df" +dependencies = [ + "difference", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f883590242d3c6fc5bf50299011695fa6590c2c70eac95ee1bdb9a733ad1a2" + +[[package]] +name = "predicates-tree" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54ff541861505aabf6ea722d2131ee980b8276e10a1297b94e896dd8b621850d" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "proc-macro2" +version = "1.0.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.8", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.8", + "rand_core 0.3.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.8", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rayon" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" + +[[package]] +name = "ryu" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "serde" +version = "1.0.152" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_cbor" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" +dependencies = [ + "half", + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.152" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "termcolor" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "termtree" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95059e91184749cb66be6dc994f67f182b6d897cb3df74a5bf66b5e709295fd8" + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "textwrap" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "unicode-ident" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" + +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" + +[[package]] +name = "web-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/courses/rust/projects/kvs_3/Cargo.toml b/courses/rust/projects/kvs_3/Cargo.toml new file mode 100644 index 000000000..dfe4b5fd6 --- /dev/null +++ b/courses/rust/projects/kvs_3/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "kvs" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dev-dependencies] +assert_cmd = "0.11" +criterion = "0.3" +predicates = "1.0.0" +rand = "0.6.5" +tempfile = "3.0.7" +walkdir = "2.2.7" + +[dependencies] +clap = "3.2.14" +failure = "0.1.8" +serde_json = "1.0" +serde = { version = "1.0", features = ["derive"] } +rand = "0.8.5" \ No newline at end of file diff --git a/courses/rust/projects/kvs_3/README.md b/courses/rust/projects/kvs_3/README.md new file mode 100644 index 000000000..356fddc19 --- /dev/null +++ b/courses/rust/projects/kvs_3/README.md @@ -0,0 +1,486 @@ +# PNA Rust Project 3: Synchronous client-server networking + +**Task**: Create a _single-threaded_, persistent key/value store _server and client +with synchronous networking over a custom protocol_. + +**Goals**: + +- Create a client-server application +- Write a custom protocol with `std` networking APIs +- Introduce logging to the server +- Implement pluggable backends with traits +- Benchmark the hand-written backend against `sled` + +**Topics**: `std::net`, logging, traits, benchmarking. + + + +- [Introduction](#user-content-introduction) +- [Project spec](#user-content-project-spec) +- [Project setup](#user-content-project-setup) +- [Part 1: Command line parsing](#user-content-part-1-command-line-parsing) +- [Part 2: Logging](#user-content-part-2-logging) +- [Part 3: Client-server networking setup](#user-content-part-3-client-server-networking-setup) +- [Part 4: Implementing commands across the network](#user-content-part-4-implementing-commands-across-the-network) +- [Part 5: Pluggable storage engines](#user-content-part-5-pluggable-storage-engines) +- [Part 6: Benchmarking](#user-content-part-6-benchmarking) + +## Introduction + +In this project you will create a simple key/value server and client. They will +communicate with a custom networking protocol of your design. You will emit logs +using standard logging crates, and handle errors correctly across the network +boundary. Once you have a working client-server architecture, +then you will abstract the storage engine behind traits, and compare +the performance of yours to the [`sled`] engine. + +## Project spec + +The cargo project, `kvs`, builds a command-line key-value store client called +`kvs-client`, and a key-value store server called `kvs-server`, both of which in +turn call into a library called `kvs`. The client speaks to the server over +a custom protocol. + +The `kvs-server` executable supports the following command line arguments: + +- `kvs-server [--addr IP-PORT] [--engine ENGINE-NAME]` + + Start the server and begin listening for incoming connections. `--addr` + accepts an IP address, either v4 or v6, and a port number, with the format + `IP:PORT`. If `--addr` is not specified then listen on `127.0.0.1:4000`. + + If `--engine` is specified, then `ENGINE-NAME` must be either "kvs", in which + case the built-in engine is used, or "sled", in which case sled is used. If + this is the first run (there is no data previously persisted) then the default + value is "kvs"; if there is previously persisted data then the default is the + engine already in use. If data was previously persisted with a different + engine than selected, print an error and exit with a non-zero exit code. + + Print an error and return a non-zero exit code on failure to bind a socket, if + `ENGINE-NAME` is invalid, if `IP-PORT` does not parse as an address. + +- `kvs-server -V` + + Print the version. + +The `kvs-client` executable supports the following command line arguments: + +- `kvs-client set [--addr IP-PORT]` + + Set the value of a string key to a string. + + `--addr` accepts an IP address, either v4 or v6, and a port number, with the + format `IP:PORT`. If `--addr` is not specified then connect on + `127.0.0.1:4000`. + + Print an error and return a non-zero exit code on server error, + or if `IP-PORT` does not parse as an address. + +- `kvs-client get [--addr IP-PORT]` + + Get the string value of a given string key. + + `--addr` accepts an IP address, either v4 or v6, and a port number, with the + format `IP:PORT`. If `--addr` is not specified then connect on + `127.0.0.1:4000`. + + Print an error and return a non-zero exit code on server error, + or if `IP-PORT` does not parse as an address. + +- `kvs-client rm [--addr IP-PORT]` + + Remove a given string key. + + `--addr` accepts an IP address, either v4 or v6, and a port number, with the + format `IP:PORT`. If `--addr` is not specified then connect on + `127.0.0.1:4000`. + + Print an error and return a non-zero exit code on server error, + or if `IP-PORT` does not parse as an address. A "key not found" is also + treated as an error in the "rm" command. + +- `kvs-client -V` + + Print the version. + +All error messages should be printed to stderr. + +The `kvs` library contains four types: + +- `KvsClient` - implements the functionality required for `kvs-client` to speak + to `kvs-server` +- `KvsServer` - implements the functionality to serve responses to `kvs-client` + from `kvs-server` +- `KvsEngine` trait - defines the storage interface called by `KvsServer` +- `KvStore` - implements by hand the `KvsEngine` trait +- `SledKvsEngine` - implements `KvsEngine` for the [`sled`] storage engine. + +[`sled`]: https://github.com/spacejam/sled + +The design of `KvsClient` and `KvsServer` are up to you, and will be informed by +the design of your network protocol. The test suite does not directly use either +type, but only exercises them via the CLI. + +The `KvsEngine` trait supports the following methods: + +- `KvsEngine::set(&mut self, key: String, value: String) -> Result<()>` + + Set the value of a string key to a string. + + Return an error if the value is not written successfully. + +- `KvsEngine::get(&mut self, key: String) -> Result>` + + Get the string value of a string key. + If the key does not exist, return `None`. + + Return an error if the value is not read successfully. + +- `KvsEngine::remove(&mut self, key: String) -> Result<()>` + + Remove a given string key. + + Return an error if the key does not exit or value is not read successfully. + +When setting a key to a value, `KvStore` writes the `set` command to disk in +a sequential log. When removing a key, `KvStore` writes the `rm` command to +the log. On startup, the commands in the log are re-evaluated and the +log pointer (file offset) of the last command to set each key recorded in the +in-memory index. + +When retrieving a value for a key with the `get` command, it searches the index, +and if found then loads from the log, and evaluates, the command at the +corresponding log pointer. + +When the size of the uncompacted log entries reach a given threshold, `KvStore` +compacts it into a new log, removing redundant entries to reclaim disk space. + + + +## Project setup + +Continuing from your previous project, delete your previous `tests` directory and +copy this project's `tests` directory into its place. This project should +contain a library named `kvs`, and two executables, `kvs-server` and +`kvs-client`. + +You need the following dev-dependencies in your `Cargo.toml`: + +```toml +[dev-dependencies] +assert_cmd = "0.11" +criterion = "0.3" +predicates = "1.0.0" +rand = "0.6.5" +tempfile = "3.0.7" +walkdir = "2.2.7" +``` + +As with previous projects, add enough definitions that the test suite builds. + + +## Part 1: Command line parsing + +There's little new about the command line parsing in this project compared to +previous projects. The `kvs-client` binary accepts the same command line +arguments as in previous projects. And now `kvs-server` has its own set of +command line arguments to handle, as described previously in the spec. + +_Stub out the `kvs-server` command line handling._ + + +## Part 2: Logging + +Production server applications tend to have robust and configurable logging. So +now we're going to add logging to `kvs-server`, and as we continue will look +for useful information to log. During development it is common to use logging +at the `debug!` and `trace!` levels for "println debugging". + +There are two prominent logging systems in Rust: [`log`] and [`slog`]. Both +export similar macros for logging at different levels, like `error!`, `info!` +etc. Both are extensible, supporting different backends, for logging to the +console, logging to file, logging to the system log, etc. + +[`log`]: https://docs.rs/log/ +[`slog`]: https://docs.rs/slog/ + +The major difference is that `log` is fairly simple, logging only formatted +strings; `slog` is feature-rich, and supports "structured logging", where log +entries are typed and serialized in easily-parsed formats. + +`log` dates from the very earliest days of Rust, where it was part of the +compiler, then part of the standard library, and finally released as its own +crate. It is maintained by the Rust Project. `slog` is newer and maintained +independently. Both are widely used. + +For both systems, one needs to select a "sink" crate, one that the logger +sends logs to for display or storage. + +_Read about both of them, choose the one that appeals to you, add them as +dependencies, then modify `kvs-server` to initialize logging on startup, prior +to command-line parsing._ Set it up to output to stderr (sending the logs +elsewhere additionally is fine, but they must go to stderr to pass the tests in +this project). + +On startup log the server's version number. Also log the configuration. For now +that means the IP address and port, and the name of the storage engine. + + +## Part 3: Client-server networking setup + +Next we're going to set up the networking. For this project you are going to be +using the basic TCP/IP networking APIs in `std::net`: [`TcpListener`] and +[`TcpStream`]. + +[`TcpListener`]: https://doc.rust-lang.org/std/net/struct.TcpListener.html +[`TcpStream`]: https://doc.rust-lang.org/std/net/struct.TcpStream.html + +For this project, the server is synchronous and single-threaded. That means that +you will listen on a socket, then accept connections, and execute and respond to +commands one at a time. In the future we will re-visit this decision multiple +times on our journey toward an asynchronous, multi-threaded, and +high-performance database. + +Think about your manual testing workflow. Now that there are two executables to +deal with, you'll need a way to run them both at the same time. If you are like +many, you will use two terminals, running `cargo run --bin kvs-server` in +one, where it runs until you press CTRL-D, and `cargo run --bin kvs-client` +in the other. + +This is a good opportunity to use the logging macros for debugging. Go ahead and +log information about every accepted connection. + +_Before thinking about the protocol, modify `kvs-server` to listen +for and accept connections, and `kvs-client` to initiate connections._ + + +## Part 4: Implementing commands across the network + +In the last project you defined the commands your database accepts, and learned +how to serialize and deserialize them to and from the log with `serde`. + + + +Now it's time to implement the key/value store over the network, remotely +executing commands that until now have been implemented within a single process. +As with the file I/O you worked on in the last project to create the log, you +will be serializing and streaming commands with the `Read` and `Write` traits. + +You are going to design a network protocol. There are a number of ways to get +data in and out of a TCP stream, and a number of decisions to make. Is it a +text-based protocol, binary? How is the data translated from its format in +memory to its format byte-stream format? Is there a single request per +connection, or many? + +Keep in mind that it must support successful results and errors, and there are +two kinds of errors now: the ones generated by your storage engine, as well as +network errors. + +All the details of the protocol are up to you. The test suite does not care at +all how the data gets from one end to the other, just that the results are +correct. + +_Write your network protocol._ + + + + + +## Part 5: Pluggable storage engines + +Your database has a storage engine, `KvStore`, implemented by you. +Now you are going to add a second storage engine. + +There are multiple reasons to do so: + +- Different workloads require different performance characteristics. Some + storage engines may work better than other based on the workload. + +- It creates a familiar framework for comparing different backends. + +- It gives us an excuse to create and work with traits. + +- It gives us an excuse to write some comparative benchmarks! + +So you are going to _extract_ a new trait, `KvsEngine`, from the `KvStore` +interface. This is a classic _refactoring_, where existing code is transformed +into a new form incrementally. When refactoring you will generally want to break +the work up into the smallest changes that will continue to build and work. + +Here is the API you need to end up with: + +- `trait KvsEngine` has `get`, `set` and `remove` methods with the same signatures + as `KvStore`. + +- `KvStore` implements `KvsEngine`, and no longer has `get`, `set` and `remove` + methods of its own. + +- There is a new implementation of `KvsEngine`, `SledKvsEngine`. You need to fill + its `get` and `set` methods using the `sled` library later. + +It's likely that you have already stubbed out the definitions for these if your +tests are building. _Now is the time to fill them in._ Break down your +refactoring into an intentional sequence of changes, and make sure the project +continues to build and pass previously-passing tests before continuing. + +As one final step, you need to consider what happens when `kvs-server` is +started with one engine, is killed, then restarted with a different engine. This +case can only result in an error, and you need to figure out how to detect the +case to report the error. The test `cli_wrong_engine` reflects this scenario. + + +## Part 6: Benchmarking + +As the course progresses we will increasingly concern ourselves with the +performance of the database, exploring the impact of different architectures. +You are encouraged to go beyond the model described herein and experiment with +your own optimizations. + +Performance work requires benchmarking, so now we're going to get started +on that. There are many ways to benchmark databases, with standard test +suites like [ycsb] and [sysbench]. In Rust benchmarking starts with +the builtin tooling, so we will start there. + +[ycsb]: https://github.com/brianfrankcooper/YCSB +[sysbench]: https://github.com/akopytov/sysbench + +Cargo supports benchmarking with `cargo bench`. The benchmarks may either be +written using Rust's built in benchmark harness, or an external one. + +The built-in harness creates benchmarks from functions with the `#[bench]` +attribute. It cannot be used on the Rust stable channel though, and is only +documented briefly in [the unstable book][tb] and the [`test` crate docs][tc]. +It is though widely used throughout the Rust ecosystem — crates that use +it, even if they compile with stable releases, do benchmarking with nightly +releases. + +[tb]: https://doc.rust-lang.org/stable/unstable-book/library-features/test.html +[tc]: https://doc.rust-lang.org/stable/test/index.html + +That system though is effectively deprecated — it is not being updated and +will seemingly never be promoted to the stable release channel. + +There are better benchmark harnesses for Rust anyway. The one you will use is +[criterion]. And you will use it to satisfy your curiosity about the +performance of your `kvs` engine compared to the `sled` engine. + +These benchmarking tools work by defining a benchmarking function, and within +that function iterating through a loop that performs the operation to be +benchmarking. The benchmarking tool will iterate as many times as it needs to in +order to know the duration of the operation with statistical significance. + +See this basic example from the criterion guide: + +```rust +fn criterion_benchmark(c: &mut Criterion) { + c.bench_function("fib 20", |b| { + b.iter(|| { + fibonacci(20) + }); + }); +} +``` + +The call to `bench_function` defines the benchmark, and the call to `iter` +defines the code that is run for the benchmark. Code before and after the call +to `iter` is not timed. + +[criterion]: https://docs.rs/criterion + +Prepare for writing benchmarks by creating a file called `benches/benches.rs`. +Like `tests/tests.rs`, cargo will automatically find this file and compile it as +a benchmark. + +Start by writing the following benchmarks: + +- `kvs_write` - With the kvs engine, write 100 values with random keys of length + 1-100000 bytes and random values of length 1-100000 bytes. + +- `sled_write`- With the sled engine, write 100 values with random keys of + length 1-100000 bytes and random values of length 1-100000 bytes. + +- `kvs_read` - With the kvs engine, read 1000 values from previously written keys, + with keys and values of random length. + +- `sled_read` - With the sled engine, read 1000 values from previously written keys, + with keys and values of random length. + +(As an alternative to writing 4 benchmarks, you may also choose to write 2 +benchmarks parameterized over the engine, as [described in the criterion +manual][pb]). + +[pb]: https://bheisler.github.io/criterion.rs/book/user_guide/benchmarking_with_inputs.html + +These are underspecified, and there's a fair bit of nuance to implementing them +in a useful way. We need to consider at least three factors: + +- What code should be timed (and be written inside the benchmark loop), and what + code should not (and be written outside the benchmark loop)? + +- How to make the loop run identically for each iteration, despite using + "random" numbers. + +- In the "read" benchmarks, how to read from the same set of "random" keys + that were written previously. + +These are all inter-related: some code needs to be carefully selected as +un-timed setup code, and the seed values for random number generators need +to be re-used appropriately. + +In all cases, operations that may return errors should assert (with `assert!`) +that they did not return an error; and in the read case, "get" operations should +assert that the key was found. + +Random numbers can be generated with the [`rand`] crate. + +[`rand`]: https://docs.rs/crate/rand/ + +Once you have your benchmarks, run them with `cargo bench`. + +_Write the above benchmarks, and compare the results between `kvs` and `sled`._ + +_Note: please run the benchmarks on an otherwise unloaded machine. Benchmark +results are very sensitive to the environment they are run in, and while the +criterion library does its best to compensate for "noise", benchmarks are best +done on a clean machine without other active processes. If you have a spare +machine just for development, use that. If not, an AWS or other cloud instance +may produce more consistent results than your local desktop._ + + + +Nice coding, friend. Enjoy a nice break. + + + + + diff --git a/courses/rust/projects/kvs_3/src/bin/kvs-client.rs b/courses/rust/projects/kvs_3/src/bin/kvs-client.rs new file mode 100644 index 000000000..b76a38700 --- /dev/null +++ b/courses/rust/projects/kvs_3/src/bin/kvs-client.rs @@ -0,0 +1,49 @@ +use std::env::current_dir; +use clap::{arg, Command}; +use kvs::{KvStore, Result}; + +fn main() -> Result<()> { + let version = env!("CARGO_PKG_VERSION"); + let matches = Command::new("Kvs") + .version(version) + .subcommands(vec![ + Command::new("set").args(vec![ + arg!([KEY]).required(true), + arg!([VALUE]).required(true), + ]), + Command::new("get").args(vec![arg!([KEY]).required(true)]), + Command::new("rm").args(vec![arg!([KEY]).required(true)]), + ]) + .get_matches(); + + let mut kv_store = KvStore::open(current_dir().unwrap())?; + + match matches.subcommand() { + Some(("set", args))=> { + let key = args.value_of("KEY").unwrap(); + let value = args.value_of("VALUE").unwrap(); + kv_store.set(key.to_string(), value.clone().to_string())?; + }, + Some(("get", args)) => { + let key = args.value_of("KEY").unwrap(); + let value = kv_store.get(key.to_string())?; + match value { + Some(x) => println!("{}", x), + None => println!("Key not found"), + } + }, + Some(("rm", args)) => { + let key = args.value_of("KEY").unwrap(); + let result = kv_store.remove(key.to_string()); + match result { + Err(err) => { + println!("{}", err); + std::process::exit(1); + }, + _ => {}, + } + }, + _ => std::process::exit(1), + } + Ok(()) +} diff --git a/courses/rust/projects/kvs_3/src/bin/kvs-server.rs b/courses/rust/projects/kvs_3/src/bin/kvs-server.rs new file mode 100644 index 000000000..dcd77a405 --- /dev/null +++ b/courses/rust/projects/kvs_3/src/bin/kvs-server.rs @@ -0,0 +1,29 @@ +use std::{net::TcpListener}; +use clap::{Command, arg}; +use kvs::{Result}; + +fn main() -> Result<()> { + let version = env!("CARGO_PKG_VERSION"); + let matches = Command::new("kvs-server") + .arg( + arg!(--addr "Provide IP:PORT" + ) + // We don't have syntax yet for optional options, so manually calling `required` + .required(false) + .default_value("127.0.0.1:4000") + + ) + .version(version) + .get_matches(); + + let address = matches.get_one::("addr").unwrap(); + + + let tcp_listener = TcpListener::bind(address).unwrap(); + for stream in tcp_listener.incoming() { + let _tcp_stream = stream.unwrap(); + print!("Connection Established!!") + } + + Ok(()) +} diff --git a/courses/rust/projects/kvs_3/src/lib.rs b/courses/rust/projects/kvs_3/src/lib.rs new file mode 100644 index 000000000..1165be503 --- /dev/null +++ b/courses/rust/projects/kvs_3/src/lib.rs @@ -0,0 +1,378 @@ +extern crate core; + +use rand::Rng; +use std::collections::HashMap; +use std::path::PathBuf; +use std::time::{SystemTime, UNIX_EPOCH}; +use failure::Fail; +use serde::{Serialize, Deserialize}; +use std::fs::{File, OpenOptions}; +use std::io; +use std::io::prelude::*; +use std::io::{SeekFrom, Seek}; +use std::io::{BufReader, BufWriter}; + +use crate::KvsCommand::{Rm, Set}; +use crate::KvsError::KeyNotFound; + +pub type Result = std::result::Result; + +#[derive(Fail, Debug)] +#[fail(display = "Error in KVS")] +pub enum KvsError { + #[fail(display = "{}", _0)] + IO(#[cause] io::Error), + #[fail(display = "Error Serializing or Deserializing {}", _0)] + SerdeError(serde_json::Error), + #[fail(display = "Key not found")] + KeyNotFound +} + +impl From for KvsError { + fn from(err: io::Error) -> Self { + return KvsError::IO(err) + } +} + +impl From for KvsError { + fn from(err: serde_json::Error) -> Self { + return KvsError::SerdeError(err) + } +} + +#[derive(Serialize, Deserialize)] +pub enum KvsCommand { + Set(String, String, u128), + Rm(String, u128) +} + +impl KvsCommand { + + pub fn set(key : String, value : String, timestamp : u128) -> KvsCommand { + Set(key, value, timestamp) + } + + pub fn rm(key : String, timestamp : u128) -> KvsCommand { + Rm(key, timestamp) + } +} +pub struct KvStore { + dir_path : PathBuf, + files : HashMap, + map : HashMap, + cur_offset : u64, + cur_index : u32 +} + +struct KvsEntry { + file_index : u32, + timestamp : u128, + offset : u64 +} + +impl KvsEntry { + + fn new(file_index : u32, timestamp : u128, offset : u64) -> Self { + KvsEntry {file_index, timestamp, offset} + } + + fn get_timestamp(&self) -> u128 { + self.timestamp + } + +} + +struct KvsFile { + file_path : PathBuf, + buf_reader : BufReader, + buf_writer : BufWriter, + index : u32 +} + +impl KvStore { + + fn get_timestamp() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap().as_millis() + } + + fn get_file_name(extension : &str) -> String { + let mut rng = rand::thread_rng(); + let n1: u8 = rng.gen(); + format!("kvs{}_{}.{}", n1, Self::get_timestamp(), extension) + } + + fn create_new_file(dir_path : &PathBuf, index : u32) -> KvsFile { + let file_name = Self::get_file_name("bin"); + let mut file_path = dir_path.clone(); + file_path.push(file_name); + Self::create_kvs_file(&file_path, index) + } + + fn find_all_file_in_dir(dir_path : &PathBuf) -> Vec { + let mut files_vec = Vec::new(); + for entry in std::fs::read_dir(dir_path).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + + let metadata = std::fs::metadata(&path).unwrap(); + if metadata.is_file() && path.extension().unwrap() == "bin" { + files_vec.push(path); + } + } + files_vec + } + + fn get_kvs_files(file_paths : Vec) -> Vec { + let mut files = vec![]; + let mut itr = 0; + for file_path in file_paths { + let kvs_file = Self::create_kvs_file(&file_path, itr); + files.push(kvs_file); + itr += 1; + } + files + } + + fn create_kvs_file(file_path : &PathBuf, index : u32) -> KvsFile { + let buf_writer = Self::create_buf_writer(file_path); + let buf_reader = Self::create_buf_reader(file_path); + KvsFile { + file_path : file_path.clone(), + buf_reader, + buf_writer, + index + } + } + + fn create_buf_reader(file_path : &PathBuf) -> BufReader { + let file = OpenOptions::new() + .read(true) + .open(file_path).unwrap(); + + let buf_reader = BufReader::new(file); + buf_reader + } + + fn create_buf_writer(file_path : &PathBuf) -> BufWriter { + let file = OpenOptions::new() + .create(true) + .append(true) + .open(file_path).unwrap(); + let buf_writer = BufWriter::new(file); + buf_writer + } + + fn read_from_file(kvs_file : &mut KvsFile) -> Result<(HashMap, HashMap)> { + let mut set_map = HashMap::new(); + let mut del_map = HashMap::new(); + + let mut itr = serde_json::Deserializer::from_reader(&mut kvs_file.buf_reader) + .into_iter::(); + + let mut offset = 0; + loop { + let kvs_command_option = itr.next(); + match kvs_command_option { + Some(kvs_command_result) => { + let kvs_command = kvs_command_result.unwrap(); + match kvs_command { + Set(key, _, time_stamp) => { + set_map.insert(key, KvsEntry::new(kvs_file.index, time_stamp, offset)); + }, + Rm(key, time_stamp) => { + del_map.insert(key, KvsEntry::new(kvs_file.index, time_stamp, offset)); + } + }; + }, + None => break + } + let itr_offset = itr.byte_offset(); + offset = itr_offset as u64; + } + kvs_file.buf_reader.seek(SeekFrom::Start(0)).unwrap(); + Ok((set_map, del_map)) + } + + pub fn open(path: impl Into) -> Result { + let dir_path = path.into(); + let file_paths = Self::find_all_file_in_dir(&dir_path); + let mut files = Self::get_kvs_files(file_paths); + let mut set_map : HashMap = HashMap::new(); + let mut del_map : HashMap = HashMap::new(); + for mut file in &mut files { + let out = Self::read_from_file(&mut file).unwrap(); + let s_map = out.0; + for s_map_entry in s_map { + if set_map.contains_key(&s_map_entry.0) { + let cur_value = set_map.get(&s_map_entry.0).unwrap(); + if s_map_entry.1.timestamp > cur_value.get_timestamp() { + set_map.insert(s_map_entry.0, s_map_entry.1); + } + } else { + set_map.insert(s_map_entry.0, s_map_entry.1); + } + } + + let d_map = out.1; + for d_map_entry in d_map { + if del_map.contains_key(&d_map_entry.0) { + let cur_value = del_map.get(&d_map_entry.0).unwrap(); + if d_map_entry.1.timestamp > cur_value.get_timestamp() { + del_map.insert(d_map_entry.0, d_map_entry.1); + } + } else { + del_map.insert(d_map_entry.0, d_map_entry.1); + } + } + } + + let mut files_map : HashMap = HashMap::new(); + for file in files { + files_map.insert(file.index, file); + } + + let mut final_map = HashMap::new(); + for entry in set_map { + final_map.insert(entry.0, entry.1); + } + + for entry in del_map { + if final_map.contains_key(&entry.0) { + let cur_val = final_map.get(&entry.0).unwrap(); + if cur_val.get_timestamp() < entry.1.timestamp { + final_map.remove(&entry.0); + } + } + } + + let cur_index = files_map.len() as u32; + let cur_file = Self::create_new_file(&dir_path, cur_index); + files_map.insert(cur_index, cur_file); + + Ok(KvStore { + dir_path, + files : files_map, + map : final_map, + cur_offset : 0, + cur_index + }) + + } + + pub fn set(&mut self, key: String, value: String) -> Result<()> { + let timestamp = Self::get_timestamp(); + let command = KvsCommand::set(key.clone(), value.clone(), timestamp); + let json_str = serde_json::to_string(&command)?; + let cur_file = self.files.get_mut(&self.cur_index).unwrap(); + let bytes_written = cur_file.buf_writer.write(json_str.as_bytes())?; + cur_file.buf_writer.flush()?; + self.map.insert(key,KvsEntry::new(cur_file.index, timestamp, self.cur_offset)); + self.cur_offset += bytes_written as u64; + self.check_cur_file_over_size(); + self.compaction_required(); + Ok(()) + } + + pub fn get(&mut self, key: String) -> Result> { + match self.map.get(&key) { + Some(entry) => { + let kvs_file = self.files.get_mut(&entry.file_index).unwrap(); + kvs_file.buf_reader.seek(SeekFrom::Start(entry.offset.clone()))?; + let kvs_command = serde_json::Deserializer::from_reader(&mut kvs_file.buf_reader) + .into_iter::().next().unwrap()?; + match kvs_command { + Set(_, v, _) => Ok(Some(v)), + _ => panic!("No Set command found") + } + } + None => Ok(None), + } + } + + pub fn remove(&mut self, key: String) -> Result<()> { + if self.map.contains_key(&key) { + let timestamp = Self::get_timestamp(); + let command = KvsCommand::rm(key.clone(), timestamp); + let json_str = serde_json::to_string(&command)?; + let cur_file = self.files.get_mut(&self.cur_index).unwrap(); + let bytes_written = cur_file.buf_writer.write(json_str.as_bytes())?; + cur_file.buf_writer.flush()?; + self.cur_offset += bytes_written as u64; + self.map.remove(&key); + self.check_cur_file_over_size(); + self.compaction_required(); + Ok(()) + } else { + Err(KeyNotFound) + } + } + + fn check_cur_file_over_size(&mut self) -> () { + if self.cur_offset >= 1_000_000 { + let file_index = self.get_file_index(); + self.files.insert(file_index, Self::create_new_file(&self.dir_path, file_index)); + self.cur_index = file_index; + self.cur_offset = 0; + } + } + + fn compaction_required(&mut self) -> () { + if self.files.len() == 4 { + self.compact().unwrap(); + } + } + + fn get_file_index(&self) -> u32 { + let mut rng = rand::thread_rng(); + loop { + let index = rng.gen::(); + if !self.files.contains_key(&index) { + return index.clone() + } + } + } + + fn compact(&mut self) -> Result<()> { + + let compaction_file_index = self.get_file_index(); + self.files.insert(compaction_file_index, Self::create_new_file(&self.dir_path, compaction_file_index)); + + let keys = self.map.keys().map(|x| x.clone()).collect::>(); + + let mut offset = 0; + for entry in keys { + let value = self.get(entry.clone()).unwrap().unwrap(); + let kvs_entry = self.map.get(&entry).unwrap(); + let command = KvsCommand::set(entry.clone(), value.clone(),kvs_entry.get_timestamp()); + let json_str = serde_json::to_string(&command)?; + let compaction_file = self.files.get_mut(&compaction_file_index).unwrap(); + let bytes_written = compaction_file.buf_writer.write(json_str.as_bytes())?; + compaction_file.buf_writer.flush()?; + self.map.insert(entry.clone(), KvsEntry::new(compaction_file_index, kvs_entry.get_timestamp(), offset)); + offset += bytes_written as u64; + } + + let cur_file_index = self.get_file_index(); + self.files.insert(cur_file_index, Self::create_new_file(&self.dir_path, cur_file_index)); + self.cur_index = cur_file_index; + self.cur_offset = 0; + + let file_to_be_removed_indexs = self.files.keys() + .map(|x| x.clone()) + .into_iter() + .collect::>(); + + for file_index in file_to_be_removed_indexs { + if file_index == compaction_file_index || file_index == cur_file_index { + // DO Nothing + } else { + let file = self.files.remove(&file_index).unwrap(); + std::fs::remove_file(file.file_path).unwrap(); + } + } + + Ok(()) + } +} diff --git a/courses/rust/projects/kvs_3/tests/cli.rs b/courses/rust/projects/kvs_3/tests/cli.rs new file mode 100644 index 000000000..da6b7207b --- /dev/null +++ b/courses/rust/projects/kvs_3/tests/cli.rs @@ -0,0 +1,337 @@ +use assert_cmd::prelude::*; +use predicates::str::{contains, is_empty}; +use std::fs::{self, File}; +use std::process::Command; +use std::sync::mpsc; +use std::thread; +use std::time::Duration; +use tempfile::TempDir; + +// `kvs-client` with no args should exit with a non-zero code. +#[test] +fn client_cli_no_args() { + let temp_dir = TempDir::new().unwrap(); + let mut cmd = Command::cargo_bin("kvs-client").unwrap(); + cmd.current_dir(&temp_dir).assert().failure(); +} + +#[test] +fn client_cli_invalid_get() { + let temp_dir = TempDir::new().unwrap(); + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["get"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["get", "extra", "field"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["get", "key", "--addr", "invalid-addr"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["get", "key", "--unknown-flag"]) + .current_dir(&temp_dir) + .assert() + .failure(); +} + +#[test] +fn client_cli_invalid_set() { + let temp_dir = TempDir::new().unwrap(); + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["set"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["set", "missing_field"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["set", "key", "value", "extra_field"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["set", "key", "value", "--addr", "invalid-addr"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["get", "key", "--unknown-flag"]) + .current_dir(&temp_dir) + .assert() + .failure(); +} + +#[test] +fn client_cli_invalid_rm() { + let temp_dir = TempDir::new().unwrap(); + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["rm"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["rm", "extra", "field"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["rm", "key", "--addr", "invalid-addr"]) + .current_dir(&temp_dir) + .assert() + .failure(); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["rm", "key", "--unknown-flag"]) + .current_dir(&temp_dir) + .assert() + .failure(); +} + +#[test] +fn client_cli_invalid_subcommand() { + let temp_dir = TempDir::new().unwrap(); + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["unknown"]) + .current_dir(&temp_dir) + .assert() + .failure(); +} + +// `kvs-client -V` should print the version +#[test] +fn client_cli_version() { + let temp_dir = TempDir::new().unwrap(); + let mut cmd = Command::cargo_bin("kvs-client").unwrap(); + cmd.args(&["-V"]) + .current_dir(&temp_dir) + .assert() + .stdout(contains(env!("CARGO_PKG_VERSION"))); +} + +// `kvs-server -V` should print the version +#[test] +fn server_cli_version() { + let temp_dir = TempDir::new().unwrap(); + let mut cmd = Command::cargo_bin("kvs-server").unwrap(); + cmd.args(&["-V"]) + .current_dir(&temp_dir) + .assert() + .stdout(contains(env!("CARGO_PKG_VERSION"))); +} + +#[test] +fn cli_log_configuration() { + let temp_dir = TempDir::new().unwrap(); + let stderr_path = temp_dir.path().join("stderr"); + let mut cmd = Command::cargo_bin("kvs-server").unwrap(); + let mut child = cmd + .args(&["--engine", "kvs", "--addr", "127.0.0.1:4001"]) + .current_dir(&temp_dir) + .stderr(File::create(&stderr_path).unwrap()) + .spawn() + .unwrap(); + thread::sleep(Duration::from_secs(1)); + child.kill().expect("server exited before killed"); + + let content = fs::read_to_string(&stderr_path).expect("unable to read from stderr file"); + assert!(content.contains(env!("CARGO_PKG_VERSION"))); + assert!(content.contains("kvs")); + assert!(content.contains("127.0.0.1:4001")); +} + +#[test] +fn cli_wrong_engine() { + // sled first, kvs second + { + let temp_dir = TempDir::new().unwrap(); + let mut cmd = Command::cargo_bin("kvs-server").unwrap(); + let mut child = cmd + .args(&["--engine", "sled", "--addr", "127.0.0.1:4002"]) + .current_dir(&temp_dir) + .spawn() + .unwrap(); + thread::sleep(Duration::from_secs(1)); + child.kill().expect("server exited before killed"); + + let mut cmd = Command::cargo_bin("kvs-server").unwrap(); + cmd.args(&["--engine", "kvs", "--addr", "127.0.0.1:4003"]) + .current_dir(&temp_dir) + .assert() + .failure(); + } + + // kvs first, sled second + { + let temp_dir = TempDir::new().unwrap(); + let mut cmd = Command::cargo_bin("kvs-server").unwrap(); + let mut child = cmd + .args(&["--engine", "kvs", "--addr", "127.0.0.1:4002"]) + .current_dir(&temp_dir) + .spawn() + .unwrap(); + thread::sleep(Duration::from_secs(1)); + child.kill().expect("server exited before killed"); + + let mut cmd = Command::cargo_bin("kvs-server").unwrap(); + cmd.args(&["--engine", "sled", "--addr", "127.0.0.1:4003"]) + .current_dir(&temp_dir) + .assert() + .failure(); + } +} + +fn cli_access_server(engine: &str, addr: &str) { + let (sender, receiver) = mpsc::sync_channel(0); + let temp_dir = TempDir::new().unwrap(); + let mut server = Command::cargo_bin("kvs-server").unwrap(); + let mut child = server + .args(&["--engine", engine, "--addr", addr]) + .current_dir(&temp_dir) + .spawn() + .unwrap(); + let handle = thread::spawn(move || { + let _ = receiver.recv(); // wait for main thread to finish + child.kill().expect("server exited before killed"); + }); + thread::sleep(Duration::from_secs(1)); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["set", "key1", "value1", "--addr", addr]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout(is_empty()); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["get", "key1", "--addr", addr]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout("value1\n"); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["set", "key1", "value2", "--addr", addr]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout(is_empty()); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["get", "key1", "--addr", addr]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout("value2\n"); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["get", "key2", "--addr", addr]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout(contains("Key not found")); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["rm", "key2", "--addr", addr]) + .current_dir(&temp_dir) + .assert() + .failure() + .stderr(contains("Key not found")); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["set", "key2", "value3", "--addr", addr]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout(is_empty()); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["rm", "key1", "--addr", addr]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout(is_empty()); + + sender.send(()).unwrap(); + handle.join().unwrap(); + + // Reopen and check value + let (sender, receiver) = mpsc::sync_channel(0); + let mut server = Command::cargo_bin("kvs-server").unwrap(); + let mut child = server + .args(&["--engine", engine, "--addr", addr]) + .current_dir(&temp_dir) + .spawn() + .unwrap(); + let handle = thread::spawn(move || { + let _ = receiver.recv(); // wait for main thread to finish + child.kill().expect("server exited before killed"); + }); + thread::sleep(Duration::from_secs(1)); + + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["get", "key2", "--addr", addr]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout(contains("value3")); + Command::cargo_bin("kvs-client") + .unwrap() + .args(&["get", "key1", "--addr", addr]) + .current_dir(&temp_dir) + .assert() + .success() + .stdout(contains("Key not found")); + sender.send(()).unwrap(); + handle.join().unwrap(); +} + +#[test] +fn cli_access_server_kvs_engine() { + cli_access_server("kvs", "127.0.0.1:4004"); +} + +#[test] +fn cli_access_server_sled_engine() { + cli_access_server("sled", "127.0.0.1:4005"); +} diff --git a/courses/rust/projects/kvs_3/tests/kv_store.rs b/courses/rust/projects/kvs_3/tests/kv_store.rs new file mode 100644 index 000000000..e1d2dad52 --- /dev/null +++ b/courses/rust/projects/kvs_3/tests/kv_store.rs @@ -0,0 +1,126 @@ +use kvs::{KvStore, KvsEngine, Result}; +use tempfile::TempDir; +use walkdir::WalkDir; + +// Should get previously stored value +#[test] +fn get_stored_value() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut store = KvStore::open(temp_dir.path())?; + + store.set("key1".to_owned(), "value1".to_owned())?; + store.set("key2".to_owned(), "value2".to_owned())?; + + assert_eq!(store.get("key1".to_owned())?, Some("value1".to_owned())); + assert_eq!(store.get("key2".to_owned())?, Some("value2".to_owned())); + + // Open from disk again and check persistent data + drop(store); + let mut store = KvStore::open(temp_dir.path())?; + assert_eq!(store.get("key1".to_owned())?, Some("value1".to_owned())); + assert_eq!(store.get("key2".to_owned())?, Some("value2".to_owned())); + + Ok(()) +} + +// Should overwrite existent value +#[test] +fn overwrite_value() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut store = KvStore::open(temp_dir.path())?; + + store.set("key1".to_owned(), "value1".to_owned())?; + assert_eq!(store.get("key1".to_owned())?, Some("value1".to_owned())); + store.set("key1".to_owned(), "value2".to_owned())?; + assert_eq!(store.get("key1".to_owned())?, Some("value2".to_owned())); + + // Open from disk again and check persistent data + drop(store); + let mut store = KvStore::open(temp_dir.path())?; + assert_eq!(store.get("key1".to_owned())?, Some("value2".to_owned())); + store.set("key1".to_owned(), "value3".to_owned())?; + assert_eq!(store.get("key1".to_owned())?, Some("value3".to_owned())); + + Ok(()) +} + +// Should get `None` when getting a non-existent key +#[test] +fn get_non_existent_value() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut store = KvStore::open(temp_dir.path())?; + + store.set("key1".to_owned(), "value1".to_owned())?; + assert_eq!(store.get("key2".to_owned())?, None); + + // Open from disk again and check persistent data + drop(store); + let mut store = KvStore::open(temp_dir.path())?; + assert_eq!(store.get("key2".to_owned())?, None); + + Ok(()) +} + +#[test] +fn remove_non_existent_key() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut store = KvStore::open(temp_dir.path())?; + assert!(store.remove("key1".to_owned()).is_err()); + Ok(()) +} + +#[test] +fn remove_key() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut store = KvStore::open(temp_dir.path())?; + store.set("key1".to_owned(), "value1".to_owned())?; + assert!(store.remove("key1".to_owned()).is_ok()); + assert_eq!(store.get("key1".to_owned())?, None); + Ok(()) +} + +// Insert data until total size of the directory decreases. +// Test data correctness after compaction. +#[test] +fn compaction() -> Result<()> { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut store = KvStore::open(temp_dir.path())?; + + let dir_size = || { + let entries = WalkDir::new(temp_dir.path()).into_iter(); + let len: walkdir::Result = entries + .map(|res| { + res.and_then(|entry| entry.metadata()) + .map(|metadata| metadata.len()) + }) + .sum(); + len.expect("fail to get directory size") + }; + + let mut current_size = dir_size(); + for iter in 0..1000 { + for key_id in 0..1000 { + let key = format!("key{}", key_id); + let value = format!("{}", iter); + store.set(key, value)?; + } + + let new_size = dir_size(); + if new_size > current_size { + current_size = new_size; + continue; + } + // Compaction triggered + + drop(store); + // reopen and check content + let mut store = KvStore::open(temp_dir.path())?; + for key_id in 0..1000 { + let key = format!("key{}", key_id); + assert_eq!(store.get(key)?, Some(format!("{}", iter))); + } + return Ok(()); + } + + panic!("No compaction detected"); +} From 48e407f1c3a052416df24642d7bb2a60cde1cf29 Mon Sep 17 00:00:00 2001 From: Sagar Arora Date: Sun, 26 Feb 2023 19:26:33 +0530 Subject: [PATCH 6/9] Adding Logging to Client and Server --- courses/rust/projects/kvs_3/Cargo.lock | 161 +++++++++++++++++- courses/rust/projects/kvs_3/Cargo.toml | 4 +- .../rust/projects/kvs_3/src/bin/kvs-client.rs | 29 +++- .../rust/projects/kvs_3/src/bin/kvs-server.rs | 16 +- 4 files changed, 201 insertions(+), 9 deletions(-) diff --git a/courses/rust/projects/kvs_3/Cargo.lock b/courses/rust/projects/kvs_3/Cargo.lock index 2e1e63a3e..714ceff7f 100644 --- a/courses/rust/projects/kvs_3/Cargo.lock +++ b/courses/rust/projects/kvs_3/Cargo.lock @@ -26,6 +26,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "assert_cmd" version = "0.11.1" @@ -109,6 +118,20 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-integer", + "num-traits", + "wasm-bindgen", + "winapi", +] + [[package]] name = "clap" version = "2.34.0" @@ -153,6 +176,22 @@ dependencies = [ "bitflags", ] +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + [[package]] name = "criterion" version = "0.3.6" @@ -253,6 +292,50 @@ dependencies = [ "memchr", ] +[[package]] +name = "cxx" +version = "1.0.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86d3488e7665a7a483b57e25bdd90d0aeb2bc7608c8d0346acf2ad3f1caf1d62" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48fcaf066a053a41a81dfb14d57d99738b767febb8b735c3016e469fac5da690" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2ef98b8b717a829ca5603af80e1f9e2e48013ab227b68ef37872ef84ee479bf" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "difference" version = "2.0.0" @@ -370,6 +453,30 @@ dependencies = [ "libc", ] +[[package]] +name = "iana-time-zone" +version = "0.1.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + [[package]] name = "indexmap" version = "1.9.2" @@ -421,11 +528,13 @@ dependencies = [ "clap 3.2.23", "criterion", "failure", + "log", "predicates", "rand 0.6.5", "rand 0.8.5", "serde", "serde_json", + "stderrlog", "tempfile", "walkdir", ] @@ -442,6 +551,15 @@ version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +[[package]] +name = "link-cplusplus" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" +dependencies = [ + "cc", +] + [[package]] name = "log" version = "0.4.17" @@ -481,6 +599,16 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg 1.1.0", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.15" @@ -837,6 +965,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scratch" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" + [[package]] name = "serde" version = "1.0.152" @@ -878,6 +1012,19 @@ dependencies = [ "serde", ] +[[package]] +name = "stderrlog" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69a26bbf6de627d389164afa9783739b56746c6c72c4ed16539f4ff54170327b" +dependencies = [ + "atty", + "chrono", + "log", + "termcolor", + "thread_local", +] + [[package]] name = "strsim" version = "0.10.0" @@ -923,9 +1070,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.2.0" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" dependencies = [ "winapi-util", ] @@ -951,6 +1098,16 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +[[package]] +name = "thread_local" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "tinytemplate" version = "1.2.1" diff --git a/courses/rust/projects/kvs_3/Cargo.toml b/courses/rust/projects/kvs_3/Cargo.toml index dfe4b5fd6..260954a40 100644 --- a/courses/rust/projects/kvs_3/Cargo.toml +++ b/courses/rust/projects/kvs_3/Cargo.toml @@ -18,4 +18,6 @@ clap = "3.2.14" failure = "0.1.8" serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } -rand = "0.8.5" \ No newline at end of file +rand = "0.8.5" +log = "0.4.17" +stderrlog = "0.5.4" diff --git a/courses/rust/projects/kvs_3/src/bin/kvs-client.rs b/courses/rust/projects/kvs_3/src/bin/kvs-client.rs index b76a38700..be9428806 100644 --- a/courses/rust/projects/kvs_3/src/bin/kvs-client.rs +++ b/courses/rust/projects/kvs_3/src/bin/kvs-client.rs @@ -1,8 +1,16 @@ use std::env::current_dir; use clap::{arg, Command}; use kvs::{KvStore, Result}; +use log::{info, error}; fn main() -> Result<()> { + + stderrlog::new() + .module(module_path!()) + .verbosity(3) + .timestamp(stderrlog::Timestamp::Millisecond) + .init().unwrap(); + let version = env!("CARGO_PKG_VERSION"); let matches = Command::new("Kvs") .version(version) @@ -14,18 +22,31 @@ fn main() -> Result<()> { Command::new("get").args(vec![arg!([KEY]).required(true)]), Command::new("rm").args(vec![arg!([KEY]).required(true)]), ]) + .arg( + arg!(--addr "Provide IP:PORT" + ) + .required(false) + .default_value("127.0.0.1:4000") + + ) .get_matches(); - let mut kv_store = KvStore::open(current_dir().unwrap())?; + info!("KVS Server version={}", version); + let address = matches.get_one::("addr").unwrap(); + info!("Using IP:PORT={} to send command.", address); + + let mut kv_store = KvStore::open(current_dir().unwrap())?; match matches.subcommand() { Some(("set", args))=> { let key = args.value_of("KEY").unwrap(); let value = args.value_of("VALUE").unwrap(); + info!("Executing SET Key={}, Value={}", key, value); kv_store.set(key.to_string(), value.clone().to_string())?; }, Some(("get", args)) => { let key = args.value_of("KEY").unwrap(); + info!("Executing GET Key={}", key); let value = kv_store.get(key.to_string())?; match value { Some(x) => println!("{}", x), @@ -34,6 +55,7 @@ fn main() -> Result<()> { }, Some(("rm", args)) => { let key = args.value_of("KEY").unwrap(); + info!("Executing RM Key={}", key); let result = kv_store.remove(key.to_string()); match result { Err(err) => { @@ -43,7 +65,10 @@ fn main() -> Result<()> { _ => {}, } }, - _ => std::process::exit(1), + _ => { + error!("Unknown Command, Exiting..."); + std::process::exit(1) + } } Ok(()) } diff --git a/courses/rust/projects/kvs_3/src/bin/kvs-server.rs b/courses/rust/projects/kvs_3/src/bin/kvs-server.rs index dcd77a405..c3863bd76 100644 --- a/courses/rust/projects/kvs_3/src/bin/kvs-server.rs +++ b/courses/rust/projects/kvs_3/src/bin/kvs-server.rs @@ -1,14 +1,20 @@ use std::{net::TcpListener}; use clap::{Command, arg}; use kvs::{Result}; +use log::{info}; fn main() -> Result<()> { + + stderrlog::new() + .module(module_path!()) + .verbosity(3) + .timestamp(stderrlog::Timestamp::Millisecond) + .init().unwrap(); + let version = env!("CARGO_PKG_VERSION"); let matches = Command::new("kvs-server") .arg( - arg!(--addr "Provide IP:PORT" - ) - // We don't have syntax yet for optional options, so manually calling `required` + arg!(--addr "Provide IP:PORT") .required(false) .default_value("127.0.0.1:4000") @@ -18,11 +24,13 @@ fn main() -> Result<()> { let address = matches.get_one::("addr").unwrap(); + info!("KVS Server version={}", version); + info!("Started Listening on IP:PORT={}", address); let tcp_listener = TcpListener::bind(address).unwrap(); for stream in tcp_listener.incoming() { let _tcp_stream = stream.unwrap(); - print!("Connection Established!!") + info!("Connected to Client...") } Ok(()) From 1d0bef973bc658d551f2b7944063e99071f6b989 Mon Sep 17 00:00:00 2001 From: Sagar Arora Date: Sun, 12 Mar 2023 21:11:22 +0530 Subject: [PATCH 7/9] Single TCP Connection --- .../rust/projects/kvs_3/src/bin/kvs-client.rs | 42 ++++++----- .../rust/projects/kvs_3/src/bin/kvs-server.rs | 72 +++++++++++++++++-- courses/rust/projects/kvs_3/src/lib.rs | 12 ++++ 3 files changed, 101 insertions(+), 25 deletions(-) diff --git a/courses/rust/projects/kvs_3/src/bin/kvs-client.rs b/courses/rust/projects/kvs_3/src/bin/kvs-client.rs index be9428806..8d46eef12 100644 --- a/courses/rust/projects/kvs_3/src/bin/kvs-client.rs +++ b/courses/rust/projects/kvs_3/src/bin/kvs-client.rs @@ -1,9 +1,10 @@ -use std::env::current_dir; +use std::io::Write; +use std::net::TcpStream; use clap::{arg, Command}; -use kvs::{KvStore, Result}; +use kvs::DbCommand; use log::{info, error}; -fn main() -> Result<()> { +fn main() -> () { stderrlog::new() .module(module_path!()) @@ -35,40 +36,43 @@ fn main() -> Result<()> { let address = matches.get_one::("addr").unwrap(); info!("Using IP:PORT={} to send command.", address); + let mut tcp_stream = TcpStream::connect(address).unwrap(); + info!("Connected to IP:PORT={}", address); + - let mut kv_store = KvStore::open(current_dir().unwrap())?; match matches.subcommand() { Some(("set", args))=> { let key = args.value_of("KEY").unwrap(); let value = args.value_of("VALUE").unwrap(); info!("Executing SET Key={}, Value={}", key, value); - kv_store.set(key.to_string(), value.clone().to_string())?; + let set_command = DbCommand::Set(key.to_string(), value.clone().to_string()); + let set_command = serde_json::to_string(&set_command).unwrap(); + tcp_stream.write(set_command.as_bytes()).unwrap(); + let result_command : DbCommand = serde_json::from_reader(tcp_stream).unwrap(); + info!("SET Result = {:?}", result_command); }, Some(("get", args)) => { let key = args.value_of("KEY").unwrap(); info!("Executing GET Key={}", key); - let value = kv_store.get(key.to_string())?; - match value { - Some(x) => println!("{}", x), - None => println!("Key not found"), - } + let get_command = DbCommand::Get(key.to_string()); + let get_command = serde_json::to_string(&get_command).unwrap(); + tcp_stream.write(get_command.as_bytes()).unwrap(); + let result_command : DbCommand = serde_json::from_reader(tcp_stream).unwrap(); + info!("GET Result = {:?}", result_command); }, Some(("rm", args)) => { let key = args.value_of("KEY").unwrap(); info!("Executing RM Key={}", key); - let result = kv_store.remove(key.to_string()); - match result { - Err(err) => { - println!("{}", err); - std::process::exit(1); - }, - _ => {}, - } + let rm_command = DbCommand::Rm(key.to_string()); + let rm_command = serde_json::to_string(&rm_command).unwrap(); + tcp_stream.write(rm_command.as_bytes()).unwrap(); + let result_command : DbCommand = serde_json::from_reader(tcp_stream).unwrap(); + info!("RM Result = {:?}", result_command); }, _ => { error!("Unknown Command, Exiting..."); std::process::exit(1) } } - Ok(()) + () } diff --git a/courses/rust/projects/kvs_3/src/bin/kvs-server.rs b/courses/rust/projects/kvs_3/src/bin/kvs-server.rs index c3863bd76..8be68b402 100644 --- a/courses/rust/projects/kvs_3/src/bin/kvs-server.rs +++ b/courses/rust/projects/kvs_3/src/bin/kvs-server.rs @@ -1,7 +1,8 @@ -use std::{net::TcpListener}; +use std::{net::TcpListener, io::Write, env::current_dir}; use clap::{Command, arg}; -use kvs::{Result}; -use log::{info}; +use kvs::{Result, DbCommand, KvStore}; +use log::{info, error}; +use serde::Deserialize; fn main() -> Result<()> { @@ -23,14 +24,73 @@ fn main() -> Result<()> { .get_matches(); let address = matches.get_one::("addr").unwrap(); - + let mut kv_store = KvStore::open(current_dir().unwrap())?; + info!("KVS Server version={}", version); info!("Started Listening on IP:PORT={}", address); let tcp_listener = TcpListener::bind(address).unwrap(); for stream in tcp_listener.incoming() { - let _tcp_stream = stream.unwrap(); - info!("Connected to Client...") + let mut tcp_stream = stream.unwrap(); + info!("Connected to Client..."); + let mut de = serde_json::Deserializer::from_reader(&tcp_stream); + let command = DbCommand::deserialize(&mut de)?; + match command { + DbCommand::Set(a, b) => { + info!("Executing Command=SET Key={}, Value={}", a, b); + let result = kv_store.set(a.to_string(), b.to_string()); + let data_to_write; + if result.is_err() { + let error = result.err().unwrap().to_string(); + let error = DbCommand::Error(error); + data_to_write = serde_json::to_string(&error).unwrap(); + error!("Error command=SET, error={:?}", error) + } else { + let command = DbCommand::SetResult(a.to_string()); + data_to_write = serde_json::to_string(&command).unwrap(); + } + tcp_stream.write(data_to_write.as_bytes()).unwrap(); + info!("Command=SET Key={}, Value={} executed successfully.", a.to_string(), b); + }, + DbCommand::Get(a) => { + info!("Executing Command=GET Key={}", a); + let result = kv_store.get(a.to_string()); + let data_to_write; + if result.is_err() { + let error = result.err().unwrap().to_string(); + let error = DbCommand::Error(error); + data_to_write = serde_json::to_string(&error).unwrap(); + error!("Error Command=GET, error={:?}", error) + } else { + let command = DbCommand::GetResult(result.unwrap().unwrap()); + data_to_write = serde_json::to_string(&command).unwrap(); + } + tcp_stream.write(data_to_write.as_bytes()).unwrap(); + info!("Command=GET Key={} executed successfully.", a); + }, + DbCommand::Rm(a) => { + info!("Executing Command=RM Key={}", a); + let result = kv_store.remove(a.to_string()); + let data_to_write; + if result.is_err() { + let error = result.err().unwrap().to_string(); + let error = DbCommand::Error(error); + data_to_write = serde_json::to_string(&error).unwrap(); + error!("Error Command=RM, error={:?}", error) + } else { + let command = DbCommand::RmResult(); + data_to_write = serde_json::to_string(&command).unwrap(); + } + tcp_stream.write(data_to_write.as_bytes()).unwrap(); + info!("Command=RM Key={} executed successfully.", a); + }, + _ => { + let command = DbCommand::Error("Unknown Command".to_owned()); + let data_to_write = serde_json::to_string(&command).unwrap(); + tcp_stream.write(data_to_write.as_bytes()).unwrap(); + info!("Command=UNKNOWN, Returning Error"); + } + } } Ok(()) diff --git a/courses/rust/projects/kvs_3/src/lib.rs b/courses/rust/projects/kvs_3/src/lib.rs index 1165be503..10336f568 100644 --- a/courses/rust/projects/kvs_3/src/lib.rs +++ b/courses/rust/projects/kvs_3/src/lib.rs @@ -40,6 +40,18 @@ impl From for KvsError { } } +#[derive(Serialize, Deserialize, Debug)] +pub enum DbCommand { + Get(String), + GetResult(String), + Set(String, String), + SetResult(String), + Rm(String), + RmResult(), + Error(String) +} + + #[derive(Serialize, Deserialize)] pub enum KvsCommand { Set(String, String, u128), From b89c8bf20a6b7a138f2c8e6cbb57d1713617cb94 Mon Sep 17 00:00:00 2001 From: Sagar Arora Date: Sat, 18 Mar 2023 20:12:27 +0530 Subject: [PATCH 8/9] Adding engine Info --- courses/rust/projects/kvs_3/Cargo.lock | 428 +++++++++--------- courses/rust/projects/kvs_3/Cargo.toml | 4 +- .../rust/projects/kvs_3/src/bin/kvs-server.rs | 17 +- courses/rust/projects/kvs_3/src/lib.rs | 111 +++-- 4 files changed, 285 insertions(+), 275 deletions(-) diff --git a/courses/rust/projects/kvs_3/Cargo.lock b/courses/rust/projects/kvs_3/Cargo.lock index 714ceff7f..b0c64c810 100644 --- a/courses/rust/projects/kvs_3/Cargo.lock +++ b/courses/rust/projects/kvs_3/Cargo.lock @@ -58,15 +58,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "autocfg" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78" -dependencies = [ - "autocfg 1.1.0", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -120,9 +111,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.23" +version = "0.4.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" dependencies = [ "iana-time-zone", "js-sys", @@ -167,15 +158,6 @@ dependencies = [ "os_str_bytes", ] -[[package]] -name = "cloudabi" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -dependencies = [ - "bitflags", -] - [[package]] name = "codespan-reporting" version = "0.11.1" @@ -230,9 +212,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -240,9 +222,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -251,11 +233,11 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.13" +version = "0.9.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" dependencies = [ - "autocfg 1.1.0", + "autocfg", "cfg-if", "crossbeam-utils", "memoffset", @@ -264,18 +246,18 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.14" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ "cfg-if", ] [[package]] name = "csv" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af91f40b7355f82b0a891f50e70399475945bb0b0da4f1700ce60761c9d3e359" +checksum = "0b015497079b9a9d69c02ad25de6c0a6edef051ea6360a327d0bd05802ef64ad" dependencies = [ "csv-core", "itoa", @@ -294,9 +276,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d3488e7665a7a483b57e25bdd90d0aeb2bc7608c8d0346acf2ad3f1caf1d62" +checksum = "9a140f260e6f3f79013b8bfc65e7ce630c9ab4388c6a89c71e07226f49487b72" dependencies = [ "cc", "cxxbridge-flags", @@ -306,9 +288,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48fcaf066a053a41a81dfb14d57d99738b767febb8b735c3016e469fac5da690" +checksum = "da6383f459341ea689374bf0a42979739dc421874f112ff26f829b8040b8e613" dependencies = [ "cc", "codespan-reporting", @@ -321,15 +303,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2ef98b8b717a829ca5603af80e1f9e2e48013ab227b68ef37872ef84ee479bf" +checksum = "90201c1a650e95ccff1c8c0bb5a343213bdd317c6e600a93075bca2eff54ec97" [[package]] name = "cxxbridge-macro" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" +checksum = "0b75aed41bb2e6367cae39e6326ef817a851db13c13e4f3263714ca3cfb8de56" dependencies = [ "proc-macro2", "quote", @@ -348,6 +330,27 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "escargot" version = "0.4.0" @@ -400,12 +403,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - [[package]] name = "getrandom" version = "0.2.8" @@ -453,6 +450,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + [[package]] name = "iana-time-zone" version = "0.1.53" @@ -483,7 +486,7 @@ version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ - "autocfg 1.1.0", + "autocfg", "hashbrown", ] @@ -496,6 +499,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "io-lifetimes" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76e86b86ae312accbf05ade23ce76b625e0e47a255712b7414037385a1c05380" +dependencies = [ + "hermit-abi 0.3.1", + "libc", + "windows-sys 0.45.0", +] + [[package]] name = "itertools" version = "0.10.5" @@ -507,9 +521,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "js-sys" @@ -530,8 +544,7 @@ dependencies = [ "failure", "log", "predicates", - "rand 0.6.5", - "rand 0.8.5", + "rand", "serde", "serde_json", "stderrlog", @@ -547,9 +560,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.139" +version = "0.2.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" [[package]] name = "link-cplusplus" @@ -560,6 +573,12 @@ dependencies = [ "cc", ] +[[package]] +name = "linux-raw-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" + [[package]] name = "log" version = "0.4.17" @@ -577,11 +596,11 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" dependencies = [ - "autocfg 1.1.0", + "autocfg", ] [[package]] @@ -605,7 +624,7 @@ version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ - "autocfg 1.1.0", + "autocfg", "num-traits", ] @@ -615,7 +634,7 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ - "autocfg 1.1.0", + "autocfg", ] [[package]] @@ -704,15 +723,15 @@ dependencies = [ [[package]] name = "predicates-core" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f883590242d3c6fc5bf50299011695fa6590c2c70eac95ee1bdb9a733ad1a2" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" [[package]] name = "predicates-tree" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54ff541861505aabf6ea722d2131ee980b8276e10a1297b94e896dd8b621850d" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" dependencies = [ "predicates-core", "termtree", @@ -720,41 +739,22 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.51" +version = "1.0.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +checksum = "1d0e1ae9e836cc3beddd63db0df682593d7e2d3d891ae8c9083d2113e1744224" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.23" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" dependencies = [ "proc-macro2", ] -[[package]] -name = "rand" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" -dependencies = [ - "autocfg 0.1.8", - "libc", - "rand_chacha 0.1.1", - "rand_core 0.4.2", - "rand_hc", - "rand_isaac", - "rand_jitter", - "rand_os", - "rand_pcg", - "rand_xorshift", - "winapi", -] - [[package]] name = "rand" version = "0.8.5" @@ -762,18 +762,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" -dependencies = [ - "autocfg 0.1.8", - "rand_core 0.3.1", + "rand_chacha", + "rand_core", ] [[package]] @@ -783,24 +773,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", + "rand_core", ] -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - [[package]] name = "rand_core" version = "0.6.4" @@ -810,73 +785,11 @@ dependencies = [ "getrandom", ] -[[package]] -name = "rand_hc" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rand_isaac" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rand_jitter" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" -dependencies = [ - "libc", - "rand_core 0.4.2", - "winapi", -] - -[[package]] -name = "rand_os" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" -dependencies = [ - "cloudabi", - "fuchsia-cprng", - "libc", - "rand_core 0.4.2", - "rdrand", - "winapi", -] - -[[package]] -name = "rand_pcg" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" -dependencies = [ - "autocfg 0.1.8", - "rand_core 0.4.2", -] - -[[package]] -name = "rand_xorshift" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" -dependencies = [ - "rand_core 0.3.1", -] - [[package]] name = "rayon" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" dependencies = [ "either", "rayon-core", @@ -884,9 +797,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.10.2" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -894,15 +807,6 @@ dependencies = [ "num_cpus", ] -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", -] - [[package]] name = "redox_syscall" version = "0.2.16" @@ -929,26 +833,31 @@ version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] - [[package]] name = "rustc-demangle" version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" +[[package]] +name = "rustix" +version = "0.36.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd5c6ff11fecd55b40746d1995a02f2eb375bf8c00d192d521ee09f42bef37bc" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.45.0", +] + [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "same-file" @@ -967,15 +876,15 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "scratch" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" [[package]] name = "serde" -version = "1.0.152" +version = "1.0.156" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "314b5b092c0ade17c00142951e50ced110ec27cea304b1037c6969246c2469a4" dependencies = [ "serde_derive", ] @@ -992,9 +901,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.156" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "d7e29c4601e36bcec74a223228dce795f4cd3616341a4af93520ca1a837c087d" dependencies = [ "proc-macro2", "quote", @@ -1003,9 +912,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" dependencies = [ "itoa", "ryu", @@ -1033,9 +942,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", @@ -1056,16 +965,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95" dependencies = [ "cfg-if", "fastrand", - "libc", "redox_syscall", - "remove_dir_all", - "winapi", + "rustix", + "windows-sys 0.42.0", ] [[package]] @@ -1079,9 +987,9 @@ dependencies = [ [[package]] name = "termtree" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95059e91184749cb66be6dc994f67f182b6d897cb3df74a5bf66b5e709295fd8" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "textwrap" @@ -1120,9 +1028,9 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unicode-width" @@ -1138,12 +1046,11 @@ checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "walkdir" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" dependencies = [ "same-file", - "winapi", "winapi-util", ] @@ -1247,3 +1154,84 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" diff --git a/courses/rust/projects/kvs_3/Cargo.toml b/courses/rust/projects/kvs_3/Cargo.toml index 260954a40..35f19b2fd 100644 --- a/courses/rust/projects/kvs_3/Cargo.toml +++ b/courses/rust/projects/kvs_3/Cargo.toml @@ -9,15 +9,15 @@ edition = "2021" assert_cmd = "0.11" criterion = "0.3" predicates = "1.0.0" -rand = "0.6.5" +rand = "0.8.5" tempfile = "3.0.7" walkdir = "2.2.7" [dependencies] clap = "3.2.14" failure = "0.1.8" +rand = "0.8.5" serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } -rand = "0.8.5" log = "0.4.17" stderrlog = "0.5.4" diff --git a/courses/rust/projects/kvs_3/src/bin/kvs-server.rs b/courses/rust/projects/kvs_3/src/bin/kvs-server.rs index 8be68b402..a2fd8aaf7 100644 --- a/courses/rust/projects/kvs_3/src/bin/kvs-server.rs +++ b/courses/rust/projects/kvs_3/src/bin/kvs-server.rs @@ -1,6 +1,6 @@ use std::{net::TcpListener, io::Write, env::current_dir}; use clap::{Command, arg}; -use kvs::{Result, DbCommand, KvStore}; +use kvs::{Result, DbCommand, KvStore, KvsEngine}; use log::{info, error}; use serde::Deserialize; @@ -14,20 +14,27 @@ fn main() -> Result<()> { let version = env!("CARGO_PKG_VERSION"); let matches = Command::new("kvs-server") - .arg( + .args(&[ + arg!(--engine <"ENGINE-NAME"> "Provide EngineName") + .required(false) + .value_parser(["kvs", "sled"]) + .default_value("kvs"), arg!(--addr "Provide IP:PORT") .required(false) .default_value("127.0.0.1:4000") - - ) + ]) .version(version) .get_matches(); let address = matches.get_one::("addr").unwrap(); - let mut kv_store = KvStore::open(current_dir().unwrap())?; + let engine = matches.get_one::("engine").unwrap(); + let cur_dir = current_dir().unwrap(); + info!("Opening Database at Location={}", &cur_dir.display()); + let mut kv_store : Box = Box::new(KvStore::open(cur_dir)?); info!("KVS Server version={}", version); info!("Started Listening on IP:PORT={}", address); + info!("KvsServer Engine={}", engine); let tcp_listener = TcpListener::bind(address).unwrap(); for stream in tcp_listener.incoming() { diff --git a/courses/rust/projects/kvs_3/src/lib.rs b/courses/rust/projects/kvs_3/src/lib.rs index 10336f568..1f2cf4259 100644 --- a/courses/rust/projects/kvs_3/src/lib.rs +++ b/courses/rust/projects/kvs_3/src/lib.rs @@ -17,6 +17,15 @@ use crate::KvsError::KeyNotFound; pub type Result = std::result::Result; +pub trait KvsEngine { + + fn set(&mut self, key: String, value: String) -> Result<()>; + + fn get(&mut self, key: String) -> Result>; + + fn remove(&mut self, key: String) -> Result<()>; +} + #[derive(Fail, Debug)] #[fail(display = "Error in KVS")] pub enum KvsError { @@ -101,6 +110,57 @@ struct KvsFile { index : u32 } +impl KvsEngine for KvStore { + + fn set(&mut self, key: String, value: String) -> Result<()> { + let timestamp = Self::get_timestamp(); + let command = KvsCommand::set(key.clone(), value.clone(), timestamp); + let json_str = serde_json::to_string(&command)?; + let cur_file = self.files.get_mut(&self.cur_index).unwrap(); + let bytes_written = cur_file.buf_writer.write(json_str.as_bytes())?; + cur_file.buf_writer.flush()?; + self.map.insert(key,KvsEntry::new(cur_file.index, timestamp, self.cur_offset)); + self.cur_offset += bytes_written as u64; + self.check_cur_file_over_size(); + self.compaction_required(); + Ok(()) + } + + fn get(&mut self, key: String) -> Result> { + match self.map.get(&key) { + Some(entry) => { + let kvs_file = self.files.get_mut(&entry.file_index).unwrap(); + kvs_file.buf_reader.seek(SeekFrom::Start(entry.offset.clone()))?; + let kvs_command = serde_json::Deserializer::from_reader(&mut kvs_file.buf_reader) + .into_iter::().next().unwrap()?; + match kvs_command { + Set(_, v, _) => Ok(Some(v)), + _ => panic!("No Set command found") + } + } + None => Ok(None), + } + } + + fn remove(&mut self, key: String) -> Result<()> { + if self.map.contains_key(&key) { + let timestamp = Self::get_timestamp(); + let command = KvsCommand::rm(key.clone(), timestamp); + let json_str = serde_json::to_string(&command)?; + let cur_file = self.files.get_mut(&self.cur_index).unwrap(); + let bytes_written = cur_file.buf_writer.write(json_str.as_bytes())?; + cur_file.buf_writer.flush()?; + self.cur_offset += bytes_written as u64; + self.map.remove(&key); + self.check_cur_file_over_size(); + self.compaction_required(); + Ok(()) + } else { + Err(KeyNotFound) + } + } +} + impl KvStore { fn get_timestamp() -> u128 { @@ -129,7 +189,9 @@ impl KvStore { let path = entry.path(); let metadata = std::fs::metadata(&path).unwrap(); - if metadata.is_file() && path.extension().unwrap() == "bin" { + if metadata.is_file() && + path.extension().is_some() && + path.extension().unwrap() == "bin" { files_vec.push(path); } } @@ -273,53 +335,6 @@ impl KvStore { } - pub fn set(&mut self, key: String, value: String) -> Result<()> { - let timestamp = Self::get_timestamp(); - let command = KvsCommand::set(key.clone(), value.clone(), timestamp); - let json_str = serde_json::to_string(&command)?; - let cur_file = self.files.get_mut(&self.cur_index).unwrap(); - let bytes_written = cur_file.buf_writer.write(json_str.as_bytes())?; - cur_file.buf_writer.flush()?; - self.map.insert(key,KvsEntry::new(cur_file.index, timestamp, self.cur_offset)); - self.cur_offset += bytes_written as u64; - self.check_cur_file_over_size(); - self.compaction_required(); - Ok(()) - } - - pub fn get(&mut self, key: String) -> Result> { - match self.map.get(&key) { - Some(entry) => { - let kvs_file = self.files.get_mut(&entry.file_index).unwrap(); - kvs_file.buf_reader.seek(SeekFrom::Start(entry.offset.clone()))?; - let kvs_command = serde_json::Deserializer::from_reader(&mut kvs_file.buf_reader) - .into_iter::().next().unwrap()?; - match kvs_command { - Set(_, v, _) => Ok(Some(v)), - _ => panic!("No Set command found") - } - } - None => Ok(None), - } - } - - pub fn remove(&mut self, key: String) -> Result<()> { - if self.map.contains_key(&key) { - let timestamp = Self::get_timestamp(); - let command = KvsCommand::rm(key.clone(), timestamp); - let json_str = serde_json::to_string(&command)?; - let cur_file = self.files.get_mut(&self.cur_index).unwrap(); - let bytes_written = cur_file.buf_writer.write(json_str.as_bytes())?; - cur_file.buf_writer.flush()?; - self.cur_offset += bytes_written as u64; - self.map.remove(&key); - self.check_cur_file_over_size(); - self.compaction_required(); - Ok(()) - } else { - Err(KeyNotFound) - } - } fn check_cur_file_over_size(&mut self) -> () { if self.cur_offset >= 1_000_000 { From 0d795609fafedfe1037b9925be865ee5221a7680 Mon Sep 17 00:00:00 2001 From: Sagar Arora Date: Sun, 19 Mar 2023 20:22:03 +0530 Subject: [PATCH 9/9] Test Cases Passing --- .../rust/projects/kvs_3/src/bin/kvs-client.rs | 77 ++++++++++++++----- .../rust/projects/kvs_3/src/bin/kvs-server.rs | 72 +++++++++++++---- 2 files changed, 118 insertions(+), 31 deletions(-) diff --git a/courses/rust/projects/kvs_3/src/bin/kvs-client.rs b/courses/rust/projects/kvs_3/src/bin/kvs-client.rs index 8d46eef12..9782e20cb 100644 --- a/courses/rust/projects/kvs_3/src/bin/kvs-client.rs +++ b/courses/rust/projects/kvs_3/src/bin/kvs-client.rs @@ -19,55 +19,90 @@ fn main() -> () { Command::new("set").args(vec![ arg!([KEY]).required(true), arg!([VALUE]).required(true), + arg!(--addr "Provide IP:PORT") + .required(false) + .default_value("127.0.0.1:4000") ]), - Command::new("get").args(vec![arg!([KEY]).required(true)]), - Command::new("rm").args(vec![arg!([KEY]).required(true)]), + Command::new("get").args( + vec![ + arg!([KEY]).required(true), + arg!(--addr "Provide IP:PORT") + .required(false) + .default_value("127.0.0.1:4000") + ]), + Command::new("rm").args( + vec![ + arg!([KEY]).required(true), + arg!(--addr "Provide IP:PORT") + .required(false) + .default_value("127.0.0.1:4000") + ]), ]) - .arg( - arg!(--addr "Provide IP:PORT" - ) - .required(false) - .default_value("127.0.0.1:4000") - - ) .get_matches(); info!("KVS Server version={}", version); - let address = matches.get_one::("addr").unwrap(); - info!("Using IP:PORT={} to send command.", address); - let mut tcp_stream = TcpStream::connect(address).unwrap(); - info!("Connected to IP:PORT={}", address); - - match matches.subcommand() { Some(("set", args))=> { let key = args.value_of("KEY").unwrap(); let value = args.value_of("VALUE").unwrap(); + + let address = args.value_of("addr").unwrap(); + let mut tcp_stream = connect_to_tcp(address); + info!("Executing SET Key={}, Value={}", key, value); let set_command = DbCommand::Set(key.to_string(), value.clone().to_string()); let set_command = serde_json::to_string(&set_command).unwrap(); + tcp_stream.write(set_command.as_bytes()).unwrap(); let result_command : DbCommand = serde_json::from_reader(tcp_stream).unwrap(); - info!("SET Result = {:?}", result_command); + info!("SET Result = {:?}", result_command); }, Some(("get", args)) => { let key = args.value_of("KEY").unwrap(); + + let address = args.value_of("addr").unwrap(); + let mut tcp_stream = connect_to_tcp(address); + info!("Executing GET Key={}", key); let get_command = DbCommand::Get(key.to_string()); let get_command = serde_json::to_string(&get_command).unwrap(); + tcp_stream.write(get_command.as_bytes()).unwrap(); let result_command : DbCommand = serde_json::from_reader(tcp_stream).unwrap(); - info!("GET Result = {:?}", result_command); + info!("GET Result = {:?}", result_command); + match result_command { + DbCommand::GetResult(x) => { + println!("{}", x); + } + DbCommand::Error(x) => { + print!("{}", x); + } + _ => { + error!("UnknownResult"); + } + } }, Some(("rm", args)) => { let key = args.value_of("KEY").unwrap(); + + let address = args.value_of("addr").unwrap(); + let mut tcp_stream = connect_to_tcp(address); + info!("Executing RM Key={}", key); let rm_command = DbCommand::Rm(key.to_string()); let rm_command = serde_json::to_string(&rm_command).unwrap(); + tcp_stream.write(rm_command.as_bytes()).unwrap(); let result_command : DbCommand = serde_json::from_reader(tcp_stream).unwrap(); - info!("RM Result = {:?}", result_command); + match result_command { + DbCommand::Error(x) => { + error!("{}", x); + std::process::exit(1) + }, + _ => { + } + } }, _ => { error!("Unknown Command, Exiting..."); @@ -76,3 +111,9 @@ fn main() -> () { } () } + +fn connect_to_tcp(address : &str) -> TcpStream { + let tcp_stream = TcpStream::connect(address).unwrap(); + info!("Connected to IP:PORT={}", address); + tcp_stream +} diff --git a/courses/rust/projects/kvs_3/src/bin/kvs-server.rs b/courses/rust/projects/kvs_3/src/bin/kvs-server.rs index a2fd8aaf7..bc5251f2b 100644 --- a/courses/rust/projects/kvs_3/src/bin/kvs-server.rs +++ b/courses/rust/projects/kvs_3/src/bin/kvs-server.rs @@ -1,6 +1,6 @@ -use std::{net::TcpListener, io::Write, env::current_dir}; +use std::{net::TcpListener, io::{Write, BufReader, BufRead}, env::current_dir, path::{Path}, fs::OpenOptions}; use clap::{Command, arg}; -use kvs::{Result, DbCommand, KvStore, KvsEngine}; +use kvs::{Result, DbCommand, KvStore, KvsEngine, KvsError}; use log::{info, error}; use serde::Deserialize; @@ -30,8 +30,21 @@ fn main() -> Result<()> { let engine = matches.get_one::("engine").unwrap(); let cur_dir = current_dir().unwrap(); info!("Opening Database at Location={}", &cur_dir.display()); - let mut kv_store : Box = Box::new(KvStore::open(cur_dir)?); + let mut kv_store : Box = Box::new(KvStore::open(&cur_dir)?); + let engine_in_file = get_existing_engine(&cur_dir.as_path()); + if let Some(x) = engine_in_file { + if let Some((_, engine_type)) = x.split_once("=") { + if !str::eq(engine_type, engine) { + std::process::exit(1); + } + } + + + } else { + update_engine_in_file(&cur_dir.as_path(), &engine).unwrap(); + } + info!("KVS Server version={}", version); info!("Started Listening on IP:PORT={}", address); info!("KvsServer Engine={}", engine); @@ -62,16 +75,13 @@ fn main() -> Result<()> { DbCommand::Get(a) => { info!("Executing Command=GET Key={}", a); let result = kv_store.get(a.to_string()); - let data_to_write; - if result.is_err() { - let error = result.err().unwrap().to_string(); - let error = DbCommand::Error(error); - data_to_write = serde_json::to_string(&error).unwrap(); - error!("Error Command=GET, error={:?}", error) - } else { - let command = DbCommand::GetResult(result.unwrap().unwrap()); - data_to_write = serde_json::to_string(&command).unwrap(); - } + let command : DbCommand; + if let Some(x) = result.unwrap() { + command = DbCommand::GetResult(x); + } else { + command = DbCommand::Error(KvsError::KeyNotFound.to_string()); + } + let data_to_write = serde_json::to_string(&command).unwrap(); tcp_stream.write(data_to_write.as_bytes()).unwrap(); info!("Command=GET Key={} executed successfully.", a); }, @@ -102,3 +112,39 @@ fn main() -> Result<()> { Ok(()) } + +fn get_existing_engine(cur_dir : &Path) -> Option { + let config_file = cur_dir.join(".config"); + + let exits = Path::new(config_file.as_path()).exists(); + if !exits { + return None; + } + + let file = OpenOptions::new() + .read(true) + .open(config_file).unwrap(); + + let buf_reader = BufReader::new(file); + buf_reader.lines() + .filter(|x| x.is_ok()) + .map(|x| x.unwrap()) + .find(|x| { + return x.starts_with("engine") + }) +} + +fn update_engine_in_file(cur_dir : &Path, engine : &str) -> Result<()> { + let config_file = cur_dir.join(".config"); + let mut line_to_write = String::new(); + line_to_write.push_str("engine="); + line_to_write.push_str(engine); + line_to_write.push('\n'); + let _op = OpenOptions::new() + .write(true) + .create(true) + .open(config_file).and_then( + |mut file| file.write(line_to_write.as_bytes()) + ).unwrap(); + Ok(()) +}