[add] sqlite memory cache (limited)

this cache can be used to cache the result of bulk requests like listing pastes and board posts
it is only used for pastes, users, and levels right now
[add] move to just for builds
[add] db::cachedb
[add] db::cachedb::CacheDB
[add] db::sql::create_db_sqlite
[chore] bump version (v0.11.6 -> v0.11.7)
This commit is contained in:
hkau 2024-03-27 13:58:04 -04:00
parent 94efc41a09
commit 26efddcca5
11 changed files with 393 additions and 40 deletions

View File

@ -3,7 +3,7 @@ name = "bundlrs"
authors = ["hkau"]
license = "MIT"
version = "0.11.6"
version = "0.11.7"
edition = "2021"
rust-version = "1.75"

View File

@ -8,24 +8,30 @@ For migration from Bundles, please see [#3](https://code.stellular.org/stellular
## Install
Bundlrs provides build scripts using [just](https://github.com/casey/just). It is required that `bun`, `just`, and obviously Rust are installed before running.
Build:
```bash
bun run build
# release
bun run build:release
# release (sqlite)
just
# release (mysql)
bun run build:release:mysql
just build mysql
# release (postgres)
bun run build:release:postgres
just build postgres
# documentation
just docs
```
Documentation is automatically built when building for release.
Run:
```bash
chmod +x ./target/debug/bundlrs && ./target/debug/bundlrs
# test
just test
# release
chmod +x ./target/release/bundlrs && ./target/release/bundlrs
just run
```
Bundlrs supports the features `sqlite`, `postgres`, and `mysql`. These features dictate which database types will be used.
@ -54,6 +60,9 @@ Environment variables:
- [Bundlrs Info Page](https://stellular.net/pub/info)
- [Markdown Info Page](https://stellular.net/pub/markdown)
- [Secondary Formatting Examples](https://stellular.net/37dbdb2096)
- [Templates Info Page](https://stellular.net/pub/templates)
- [SSM Info Page](https://stellular.net/pub/ssm)
- [API Docs](https://stellular.net/api/docs/bundlrs/index.html)
## Boards

16
justfile Normal file
View File

@ -0,0 +1,16 @@
build database="sqlite":
just docs
./scripts/download_styles.sh
cargo build -r --no-default-features --features {{database}}
docs:
cargo doc --no-deps --document-private-items
test:
just docs
bun run static_build.ts
cargo run
run:
chmod +x ./target/release/bundlrs
./target/release/bundlrs

View File

@ -9,13 +9,6 @@
"peerDependencies": {
"typescript": "^5.0.0"
},
"scripts": {
"build": "./scripts/download_styles.sh && cargo build",
"build:release": "./scripts/download_styles.sh && cargo build -r --no-default-features --features sqlite",
"build:release:mysql": "./scripts/download_styles.sh && cargo build -r --no-default-features --features mysql",
"build:release:postgres": "./scripts/download_styles.sh && cargo build -r --no-default-features --features postgres",
"build:docs": "cargo doc --no-deps --document-private-items"
},
"dependencies": {
"@codemirror/autocomplete": "^6.12.0",
"@codemirror/commands": "^6.3.3",

View File

@ -1,6 +1,9 @@
//! # BundlesDB
//! Database handler for all database types
use super::sql::{self, Database, DatabaseOpts};
use super::{
cachedb::CacheDB,
sql::{self, Database, DatabaseOpts},
};
use sqlx::{Column, Row};
@ -236,6 +239,7 @@ pub struct Notification {
pub struct BundlesDB {
pub db: Database<sqlx::PgPool>,
pub options: DatabaseOpts,
pub cachedb: CacheDB,
}
#[derive(Clone)]
@ -243,6 +247,7 @@ pub struct BundlesDB {
pub struct BundlesDB {
pub db: Database<sqlx::MySqlPool>,
pub options: DatabaseOpts,
pub cachedb: CacheDB,
}
#[derive(Clone)]
@ -250,6 +255,7 @@ pub struct BundlesDB {
pub struct BundlesDB {
pub db: Database<sqlx::SqlitePool>,
pub options: DatabaseOpts,
pub cachedb: CacheDB,
}
impl BundlesDB {
@ -257,11 +263,13 @@ impl BundlesDB {
return BundlesDB {
db: sql::create_db(options.clone()).await,
options,
cachedb: CacheDB::new().await,
};
}
pub async fn init(&self) {
// ...
self.cachedb.init().await; // init cache
// create tables
let c = &self.db.client;
@ -569,6 +577,39 @@ impl BundlesDB {
&self,
username: String,
) -> DefaultReturn<Option<FullUser<String>>> {
// check if user already exists in cache
let cached = self.cachedb.get(format!("user:{}", username)).await;
if cached.is_some() {
// ...
let user = serde_json::from_str::<UserState<String>>(cached.unwrap().as_str()).unwrap();
// get role
let role = user.role.clone();
if role == "banned" {
// account banned - we're going to act like it simply does not exist
return DefaultReturn {
success: false,
message: String::from("User is banned"),
payload: Option::None,
};
}
// fetch level from role
let level = self.get_level_by_role(role.clone()).await;
// ...
return DefaultReturn {
success: true,
message: String::from("User exists (cache)"),
payload: Option::Some(FullUser {
user,
level: level.payload.level,
}),
};
}
// ...
let query: &str = if (self.db._type == "sqlite") | (self.db._type == "mysql") {
"SELECT * FROM \"Users\" WHERE \"username\" = ?"
} else {
@ -606,23 +647,33 @@ impl BundlesDB {
// fetch level from role
let level = self.get_level_by_role(role.clone()).await;
// return
// store in cache
let meta = row.get("metadata");
let user = UserState {
username: row.get("username").unwrap().to_string(),
id_hashed: row.get("id_hashed").unwrap().to_string(),
role,
timestamp: row.get("timestamp").unwrap().parse::<u128>().unwrap(),
metadata: if meta.is_some() {
meta.unwrap().to_string()
} else {
String::new()
},
};
self.cachedb
.set(
format!("user:{}", username),
serde_json::to_string::<UserState<String>>(&user).unwrap(),
)
.await;
// return
return DefaultReturn {
success: true,
message: String::from("User exists"),
message: String::from("User exists (new)"),
payload: Option::Some(FullUser {
user: UserState {
username: row.get("username").unwrap().to_string(),
id_hashed: row.get("id_hashed").unwrap().to_string(),
role,
timestamp: row.get("timestamp").unwrap().parse::<u128>().unwrap(),
metadata: if meta.is_some() {
meta.unwrap().to_string()
} else {
String::new()
},
},
user,
level: level.payload.level,
}),
};
@ -633,6 +684,18 @@ impl BundlesDB {
/// # Arguments:
/// * `name` - `String` of the level's role name
pub async fn get_level_by_role(&self, name: String) -> DefaultReturn<RoleLevelLog> {
// check if level already exists in cache
let cached = self.cachedb.get(format!("level:{}", name)).await;
if cached.is_some() {
return DefaultReturn {
success: true,
message: String::from("Level exists (cache)"),
payload: serde_json::from_str::<RoleLevelLog>(cached.unwrap().as_str()).unwrap(),
};
}
// ...
let query: &str = if (self.db._type == "sqlite") | (self.db._type == "mysql") {
"SELECT * FROM \"Logs\" WHERE \"logtype\" = 'level' AND \"content\" LIKE ?"
} else {
@ -664,14 +727,23 @@ impl BundlesDB {
let row = res.unwrap();
let row = self.textify_row(row).data;
// return
// store in cache
let id = row.get("id").unwrap().to_string();
let level = serde_json::from_str::<RoleLevel>(row.get("content").unwrap()).unwrap();
let level = RoleLevelLog { id, level };
self.cachedb
.set(
format!("level:{}", name),
serde_json::to_string::<RoleLevelLog>(&level).unwrap(),
)
.await;
// return
return DefaultReturn {
success: true,
message: String::from("Level exists"),
payload: RoleLevelLog { id, level },
message: String::from("Level exists (new)"),
payload: level,
};
}
@ -784,8 +856,9 @@ impl BundlesDB {
};
let c = &self.db.client;
let meta = &serde_json::to_string(&metadata).unwrap();
let res = sqlx::query(query)
.bind::<&String>(&serde_json::to_string(&metadata).unwrap())
.bind::<&String>(meta)
.bind::<&String>(&name)
.execute(c)
.await;
@ -798,6 +871,23 @@ impl BundlesDB {
};
}
// update cache
let existing_in_cache = self.cachedb.get(format!("user:{}", name)).await;
if existing_in_cache.is_some() {
let mut user =
serde_json::from_str::<UserState<String>>(&existing_in_cache.unwrap()).unwrap();
user.metadata = meta.to_string(); // update metadata
// update cache
self.cachedb
.update(
format!("user:{}", name),
serde_json::to_string::<UserState<String>>(&user).unwrap(),
)
.await;
}
// return
return DefaultReturn {
success: true,
@ -858,6 +948,7 @@ impl BundlesDB {
};
let c = &self.db.client;
// TODO: some kind of bulk cache update to handle this
let res = sqlx::query(query)
.bind::<&String>(
&serde_json::to_string::<PasteMetadata>(&PasteMetadata {
@ -889,6 +980,23 @@ impl BundlesDB {
};
}
// update cache
let existing_in_cache = self.cachedb.get(format!("user:{}", name)).await;
if existing_in_cache.is_some() {
let mut user =
serde_json::from_str::<UserState<String>>(&existing_in_cache.unwrap()).unwrap();
user.role = String::from("banned"); // update role
// update cache
self.cachedb
.update(
format!("user:{}", name),
serde_json::to_string::<UserState<String>>(&user).unwrap(),
)
.await;
}
// return
return DefaultReturn {
success: true,
@ -1169,6 +1277,33 @@ impl BundlesDB {
query: &str,
selector: &str,
) -> DefaultReturn<Option<FullPaste<PasteMetadata, String>>> {
// check if paste already exists in cache
let cached = self.cachedb.get(format!("paste:{}", selector)).await;
if cached.is_some() {
// ...
let paste =
serde_json::from_str::<Paste<PasteMetadata>>(cached.unwrap().as_str()).unwrap();
// get user
let user = if paste.metadata.owner.len() > 0 {
// TODO: maybe don't clone here
(self
.get_user_by_username(paste.clone().metadata.owner)
.await)
.payload
} else {
Option::None
};
// return
return DefaultReturn {
success: true,
message: String::from("Paste exists (cache)"),
payload: Option::Some(FullPaste { paste, user }),
};
}
// fetch from db
let c = &self.db.client;
let res = sqlx::query(query)
@ -1209,6 +1344,14 @@ impl BundlesDB {
views: views.to_owned(),
};
// store in cache
self.cachedb
.set(
format!("paste:{}", paste.custom_url),
serde_json::to_string::<Paste<PasteMetadata>>(&paste).unwrap(),
)
.await;
// get user
let user = if paste.metadata.owner.len() > 0 {
// TODO: maybe don't clone here
@ -1662,13 +1805,16 @@ impl BundlesDB {
"UPDATE \"Pastes\" SET (\"content\", \"content_html\", \"edit_password\", \"custom_url\", \"edit_date\") = ($1, $2, $3, $4, $5) WHERE \"custom_url\" = $6"
};
let content_html = &crate::markdown::render::parse_markdown(&content);
let edit_date = &utility::unix_epoch_timestamp().to_string();
let c = &self.db.client;
let res = sqlx::query(query)
.bind::<&String>(&content)
.bind::<&String>(&crate::markdown::render::parse_markdown(&content))
.bind::<&String>(content_html)
.bind::<&String>(&edit_password_hash)
.bind::<&String>(&custom_url)
.bind::<&String>(&utility::unix_epoch_timestamp().to_string()) // update edit_date
.bind::<&String>(edit_date) // update edit_date
.bind::<&String>(&url)
.execute(c)
.await;
@ -1681,6 +1827,28 @@ impl BundlesDB {
};
}
// update cache
let existing_in_cache = self.cachedb.get(format!("paste:{}", url)).await;
if existing_in_cache.is_some() {
let mut paste =
serde_json::from_str::<Paste<PasteMetadata>>(&existing_in_cache.unwrap()).unwrap();
paste.content = content; // update content
paste.content_html = content_html.to_string(); // update content_html
paste.edit_password = edit_password_hash; // update edit_password
paste.edit_date = edit_date.parse::<u128>().unwrap(); // update edit_date
paste.custom_url = custom_url.to_string(); // update custom_url
// update cache
self.cachedb
.update(
format!("paste:{}", url),
serde_json::to_string::<Paste<PasteMetadata>>(&paste).unwrap(),
)
.await;
}
// return
return DefaultReturn {
success: true,
@ -1763,6 +1931,23 @@ impl BundlesDB {
};
}
// update cache
let existing_in_cache = self.cachedb.get(format!("paste:{}", url)).await;
if existing_in_cache.is_some() {
let mut paste =
serde_json::from_str::<Paste<PasteMetadata>>(&existing_in_cache.unwrap()).unwrap();
paste.metadata = metadata; // update metadata
// update cache
self.cachedb
.update(
format!("paste:{}", url),
serde_json::to_string::<Paste<PasteMetadata>>(&paste).unwrap(),
)
.await;
}
// return
return DefaultReturn {
success: true,
@ -1815,6 +2000,24 @@ impl BundlesDB {
)
.await;
// update cache
let existing_in_cache = self.cachedb.get(format!("paste:{}", url)).await;
if existing_in_cache.is_some() {
let mut paste =
serde_json::from_str::<Paste<PasteMetadata>>(&existing_in_cache.unwrap())
.unwrap();
paste.views += 1;
// update cache
self.cachedb
.update(
format!("paste:{}", url),
serde_json::to_string::<Paste<PasteMetadata>>(&paste).unwrap(),
)
.await;
}
// return
return DefaultReturn {
success: true,
@ -1928,6 +2131,9 @@ impl BundlesDB {
};
}
// update cache
self.cachedb.remove(format!("paste:{}", url)).await;
// return
return DefaultReturn {
success: true,

125
src/db/cachedb.rs Normal file
View File

@ -0,0 +1,125 @@
//! # CacheDB
//!
//! In-memory shared cache SQLite database connection for caching.
//! This should be relatively easy to convert to a Redis cache if needed since this is already essentially just a key-value store.
//!
//! Identifiers should be a string following this format: `TYPE_OF_OBJECT:OBJECT_ID`. For pastes this would look like: `paste:{custom_url}`
use super::sql::{self, Database};
use sqlx::Row;
#[derive(Clone)]
pub struct CacheDB {
pub db: Database<sqlx::SqlitePool>,
}
impl CacheDB {
pub async fn new() -> CacheDB {
return CacheDB {
db: sql::create_db_sqlite("sqlite://:memory:?cache=shared").await,
};
}
pub async fn init(&self) {
// create tables
let c = &self.db.client;
let _ = sqlx::query(
"CREATE TABLE IF NOT EXISTS \"CacheObjects\" (
id VARCHAR(1000000),
content VARCHAR(1000000)
)",
)
.execute(c)
.await;
}
// GET
/// Get a cache object by its identifier
///
/// # Arguments:
/// * `id` - `String` of the object's id
pub async fn get(&self, id: String) -> Option<String> {
// fetch from database
let c = &self.db.client;
let res = sqlx::query("SELECT * FROM \"CacheObjects\" WHERE \"id\" = ?")
.bind::<&String>(&id)
.fetch_one(c)
.await;
if res.is_err() {
return Option::None;
}
// get content
let row = res.unwrap();
let content = row.get::<String, &str>("content");
// return
Option::Some(content)
}
// SET
/// Set a cache object by its identifier and content
///
/// # Arguments:
/// * `id` - `String` of the object's id
/// * `content` - `String` of the object's content
pub async fn set(&self, id: String, content: String) -> bool {
// set
let c = &self.db.client;
let res = sqlx::query("INSERT INTO \"CacheObjects\" VALUES (?, ?)")
.bind::<&String>(&id)
.bind::<&String>(&content)
.execute(c)
.await;
if res.is_err() {
return false;
}
// return
true
}
/// Update a cache object by its identifier and content
///
/// # Arguments:
/// * `id` - `String` of the object's id
/// * `content` - `String` of the object's content
pub async fn update(&self, id: String, content: String) -> bool {
// update
let c = &self.db.client;
let res = sqlx::query("UPDATE \"CacheObjects\" SET \"content\" = ? WHERE \"id\" = ?")
.bind::<&String>(&content)
.bind::<&String>(&id)
.execute(c)
.await;
if res.is_err() {
return false;
}
// return
true
}
/// Remove a cache object by its identifier
///
/// # Arguments:
/// * `id` - `String` of the object's id
pub async fn remove(&self, id: String) -> bool {
// remove
let c = &self.db.client;
let res = sqlx::query("DELETE FROM \"CacheObjects\" WHERE \"id\" = ?")
.bind::<&String>(&id)
.execute(c)
.await;
if res.is_err() {
return false;
}
// return
true
}
}

View File

@ -1,3 +1,4 @@
//! Database handling
pub mod bundlesdb;
pub mod cachedb;
pub mod sql;

View File

@ -5,7 +5,6 @@ pub struct DatabaseOpts {
pub user: String,
pub pass: String,
pub name: String,
pub cache_enabled: Option<String>,
}
// ...
@ -87,8 +86,13 @@ pub async fn create_db(options: DatabaseOpts) -> Database<sqlx::PgPool> {
#[cfg(feature = "sqlite")]
/// Create a new "sqlite" database
pub async fn create_db(_options: DatabaseOpts) -> Database<sqlx::SqlitePool> {
create_db_sqlite("sqlite://bundlrs.db").await // TODO: make allow a different connection uri?
}
/// Create a new "sqlite" database
pub async fn create_db_sqlite(url: &str) -> Database<sqlx::SqlitePool> {
// sqlite
let client = sqlx::SqlitePool::connect("sqlite://bundlrs.db").await;
let client = sqlx::SqlitePool::connect(url).await;
if client.is_err() {
panic!("Failed to connect to database!");

View File

@ -72,7 +72,6 @@ async fn main() -> std::io::Result<()> {
} else {
String::new()
},
cache_enabled: config::get_var("CACHE_ENABLED"),
})
.await;

View File

@ -284,7 +284,7 @@ fn EditPaste(props: &EditProps) -> Html {
<script type="module">
{format!("import {{ create_editor }} from \"/static/js/AtomicEditor.js\";
create_editor(document.getElementById('_doc'), '{}', '{}');
globalThis.AtomicEditor.Update(`{}`)", &props.custom_url, &props.file.path, &props.file.content.replace("`", "\\`").replace("$", "\\$").replace("\\", "\\\\"))}
globalThis.AtomicEditor.Update(`{}`)", &props.custom_url, &props.file.path, &props.file.content.replace("\\", "\\\\").replace("`", "\\`").replace("$", "\\$"))}
</script>
<style>

View File

@ -397,7 +397,7 @@ fn ProfileView(props: &Props) -> Html {
maxlength=\"200000\"
required
>{}</textarea>
</form>", props.user.username, meta.about.replace("\"", "\\\""))
</form>", props.user.username, meta.about)
} else {
// just show about
crate::markdown::render::parse_markdown(&meta.about.clone())