feat: Create database integration tests for atuin-server (#3514)

## Why?
I am trying to write a mysql database driver for atuin-server, and after
doing mostly copy-pasta on the sql queries, I need to get them working
properly. I found it was taking a long time to spin up a server and go
through and try to exercise all the functions. So enter integration
tests for a database.

By default this one test will use the `sqlite::memory:` DB URI and will
go through a number of things that I think work most of the sqlite
crate. You can also set the DB URI with an environment variable, e.g.

```shell
ATUIN_TEST_DB_URI=postgres://postgres:pg@localhost/atuin_test_ cargo test
    Finished `test` profile [unoptimized + debuginfo] target(s) in 0.47s
     Running tests/db_story.rs (/Users/joxley/projects/atuin/target/debug/deps/db_story-035ff5d9f7ad8b5e)

running 1 test
test test_full_db_story ... ok

test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.37s
```

## Questions to reviewer
1. Should I care about history?
2. Is it worth running these in an integration test at build time? If
so, I can do another PR that adds a Github workflow
3. Right now there is a single "story" test. This is GoodEnough™️ for me
to build the MySQL driver. Do you want me to make this better and if so,
how?

## Checks
- [X] I am happy for maintainers to push small adjustments to this PR,
to speed up the review cycle
- [X] I have checked that there are no existing pull requests for the
same thing

.gitignore

@@ -8,6 +8,7 @@ result
 publish.sh
 .envrc
 .planning/
+.antigravitycli/
 
 ui/backend/target
 ui/backend/gen

Cargo.lock

@@ -139,7 +139,7 @@ checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072"
 dependencies = [
  "base64ct",
  "blake2",
- "cpufeatures",
+ "cpufeatures 0.2.17",
  "password-hash",
 ]
 
@@ -830,7 +830,18 @@ checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818"
 dependencies = [
  "cfg-if",
  "cipher",
- "cpufeatures",
+ "cpufeatures 0.2.17",
+]
+
+[[package]]
+name = "chacha20"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601"
+dependencies = [
+ "cfg-if",
+ "cpufeatures 0.3.0",
+ "rand_core 0.10.1",
 ]
 
 [[package]]
@@ -1055,6 +1066,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "cpufeatures"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "crc"
 version = "3.4.0"
@@ -1186,7 +1206,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be"
 dependencies = [
  "cfg-if",
- "cpufeatures",
+ "cpufeatures 0.2.17",
  "curve25519-dalek-derive",
  "digest",
  "fiat-crypto",
@@ -1318,6 +1338,12 @@ dependencies = [
  "syn 2.0.117",
 ]
 
+[[package]]
+name = "deunicode"
+version = "1.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04"
+
 [[package]]
 name = "diff"
 version = "0.1.13"
@@ -1588,6 +1614,18 @@ dependencies = [
  "once_cell",
 ]
 
+[[package]]
+name = "fake"
+version = "5.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea6be833b323a56361118a747470a45a1bcd5c52a2ec9b1e40c83dafe687e453"
+dependencies = [
+ "deunicode",
+ "either",
+ "rand 0.10.1",
+ "time",
+]
+
 [[package]]
 name = "fancy-regex"
 version = "0.11.0"
@@ -1899,6 +1937,7 @@ dependencies = [
  "cfg-if",
  "libc",
  "r-efi 6.0.0",
+ "rand_core 0.10.1",
  "wasip2",
  "wasip3",
 ]
@@ -3455,7 +3494,7 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf"
 dependencies = [
- "cpufeatures",
+ "cpufeatures 0.2.17",
  "opaque-debug",
  "universal-hash",
 ]
@@ -3726,6 +3765,17 @@ dependencies = [
  "rand_core 0.9.5",
 ]
 
+[[package]]
+name = "rand"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207"
+dependencies = [
+ "chacha20 0.10.0",
+ "getrandom 0.4.2",
+ "rand_core 0.10.1",
+]
+
 [[package]]
 name = "rand_chacha"
 version = "0.3.1"
@@ -3764,6 +3814,12 @@ dependencies = [
  "getrandom 0.3.4",
 ]
 
+[[package]]
+name = "rand_core"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
+
 [[package]]
 name = "rand_xoshiro"
 version = "0.7.0"
@@ -4200,7 +4256,7 @@ dependencies = [
  "base64",
  "base64ct",
  "blake2",
- "chacha20",
+ "chacha20 0.9.1",
  "cipher",
  "curve25519-dalek",
  "digest",
@@ -4221,7 +4277,7 @@ checksum = "a61ebc4c4e058565285d54a1134a4f9f5d951ef901194c79437f3ee4f2538e49"
 dependencies = [
  "base64",
  "blake2",
- "chacha20",
+ "chacha20 0.9.1",
  "digest",
  "ed25519-dalek",
  "hex",
@@ -4461,7 +4517,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
 dependencies = [
  "cfg-if",
- "cpufeatures",
+ "cpufeatures 0.2.17",
  "digest",
 ]
 
@@ -4472,7 +4528,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
 dependencies = [
  "cfg-if",
- "cpufeatures",
+ "cpufeatures 0.2.17",
  "digest",
 ]
 
@@ -4584,6 +4640,12 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "snowflake_uid"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "167e0bc9777318b7b2168046f75fa9cdd1211c37cc5079847f3fec58088a0853"
+
 [[package]]
 name = "socket2"
 version = "0.6.3"
@@ -5036,6 +5098,30 @@ dependencies = [
  "log",
 ]
 
+[[package]]
+name = "tests-database"
+version = "18.16.1"
+dependencies = [
+ "async-trait",
+ "atuin-common",
+ "atuin-server-database",
+ "atuin-server-postgres",
+ "atuin-server-sqlite",
+ "eyre",
+ "fake",
+ "futures-util",
+ "metrics",
+ "regex",
+ "serde",
+ "snowflake_uid",
+ "sqlx",
+ "time",
+ "tokio",
+ "tracing",
+ "url",
+ "uuid",
+]
+
 [[package]]
 name = "thiserror"
 version = "1.0.69"

crates/atuin-server-database/src/lib.rs

@@ -74,7 +74,7 @@ impl DbSettings {
     pub fn db_type(&self) -> DbType {
         if self.db_uri.starts_with("postgres://") || self.db_uri.starts_with("postgresql://") {
             DbType::Postgres
-        } else if self.db_uri.starts_with("sqlite://") {
+        } else if self.db_uri.starts_with("sqlite:") {
             DbType::Sqlite
         } else {
             DbType::Unknown

crates/tests-database/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "tests-database"
+edition = "2024"
+description = "Integration tests for atuin-server-database and it's flavours"
+
+version = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+
+[dependencies]
+atuin-common = { workspace = true }
+atuin-server-database = { workspace = true }
+atuin-server-postgres = { workspace = true }
+atuin-server-sqlite = { workspace = true }
+
+eyre = { workspace = true }
+tokio = { workspace = true }
+tracing = { workspace = true }
+time = { workspace = true }
+serde = { workspace = true }
+sqlx = { workspace = true, features = ["sqlite", "regexp"] }
+async-trait = { workspace = true }
+uuid = { workspace = true }
+metrics = "0.24"
+futures-util = "0.3"
+
+fake = { version = "5", features = ["time"] }
+snowflake_uid = { version = "0.1.1" }
+
+regex.workspace = true
+url = "2.5.8"

crates/tests-database/README.md

@@ -0,0 +1,58 @@
+# Database Integration Tests
+
+tl;dr Run `cargo test` to exercise the SQLite database using a file in the temp directory
+
+This is an integration test suite that runs through a simple "story" of creating a user, adding some history, adding some records and doing some deleting.  The idea is to ensure that a database implementation does the right thing.
+
+## `ATUIN_TEST_DB_URI`
+Setting this will create a database at that URI and then run through the tests.  Leaving it unset will create a sqlite db in $TMP
+
+There will be a [snowflake_uid](https://docs.rs/snowflake_uid/latest/snowflake_uid/) appended to the end of the URL so the DB will be unique for the test
+
+## Postgres
+
+```shell
+ATUIN_TEST_DB_URI=postgres://postgres:pg@localhost/atuin_test_ cargo test
+```
+
+Will create a database and run the tests on it.
+
+If you want to quickly and easily run a postgres instance 
+
+```shell
+podman run --name atuin-pg-test -e POSTGRES_PASSWORD=pg -p 5432:5432 postgres
+```
+
+## Help, I want to see what was left over
+
+Set the environment variable `ATUIN_TEST_DB_NO_DESTROY` to anything and run the tests, e.g.
+
+```shell
+ATUIN_TEST_DB_NO_DESTROY=1 ATUIN_TEST_DB_URI=postgres://postgres:pg@localhost/atuin_test_ cargo test
+```
+
+will give
+
+```plain
+postgres@127.0.0.1:postgres> \l
++--------------------------------+----------+----------+------------+------------+-----------------------+
+| Name                           | Owner    | Encoding | Collate    | Ctype      | Access privileges     |
+|--------------------------------+----------+----------+------------+------------+-----------------------|
+| atuin_test_3734160738059550720 | postgres | UTF8     | en_US.utf8 | en_US.utf8 | <null>                |
+```
+
+## Design Decision
+This is intentionally not an integration test in `atuin-server-database` because it calls into the database implementation crates and I did not want to introduce a circular dependency
+
+## Glossary
+
+
+| Term | Definition |
+| :--- | :--- |
+| user | You, yes you looking at this.  It's you! |
+| session | a working session |
+| record | ... | 
+| history | not sure what the difference between this and record is |
+| store | shrug |
+
+

crates/tests-database/src/helpers.rs

@@ -0,0 +1,77 @@
+use std::env::{self, temp_dir};
+
+use atuin_server_database::{DbSettings, DbType};
+use snowflake_uid::{Config, Generator};
+use sqlx::migrate::MigrateDatabase;
+use url::Url;
+
+fn get_settings(env_uri: Option<String>) -> eyre::Result<DbSettings> {
+    let db_uri = env_uri.unwrap_or_else(|| {
+        let dir = temp_dir();
+        let file = dir.join("atuin_test_db_");
+        let filename = file.to_str().unwrap();
+        format!("sqlite://{filename}")
+    });
+
+    let mut url = Url::parse(&db_uri)?;
+    let cfg = Config::default();
+    let mut generator = Generator::from(cfg, 0);
+    let snowflake = generator.get();
+
+    let unique_path = format!("{}{snowflake}", url.path());
+    url.set_path(&unique_path);
+
+    let db_uri = url.to_string();
+
+    Ok(DbSettings {
+        db_uri,
+        read_db_uri: None,
+    })
+}
+
+#[allow(dead_code)]
+pub async fn create_test_db() -> eyre::Result<DbSettings> {
+    let var = env::var("ATUIN_TEST_DB_URI").ok();
+    let settings = get_settings(var)?;
+
+    match settings.db_type() {
+        DbType::Postgres => sqlx::Postgres::create_database(&settings.db_uri).await?,
+        DbType::Sqlite => sqlx::Sqlite::create_database(&settings.db_uri).await?,
+        atuin_server_database::DbType::Unknown => todo!(),
+    };
+
+    Ok(settings)
+}
+
+#[allow(dead_code)]
+pub async fn destroy_test_db(settings: &DbSettings) -> eyre::Result<()> {
+    match settings.db_type() {
+        DbType::Postgres => sqlx::Postgres::drop_database(&settings.db_uri).await?,
+        DbType::Sqlite => sqlx::Sqlite::drop_database(&settings.db_uri).await?,
+        DbType::Unknown => todo!(),
+    };
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use regex::Regex;
+
+    #[test]
+    fn test_settings_none() -> eyre::Result<()> {
+        let settings = get_settings(None)?;
+        let re = Regex::new(r"sqlite://.*[\\/]atuin_test_db_\d+").unwrap();
+        assert!(re.is_match(&settings.db_uri), "{}", &settings.db_uri);
+        Ok(())
+    }
+
+    #[test]
+    fn test_settings_with_param() -> eyre::Result<()> {
+        let settings = get_settings(Some("postgres://user:pass@host/database_?mode=ssl".into()))?;
+        let re = Regex::new(r"postgres://user:pass@host/database_\d+\?mode=ssl")?;
+        assert!(re.is_match(&settings.db_uri), "{}", &settings.db_uri);
+
+        Ok(())
+    }
+}

crates/tests-database/src/lib.rs

@@ -0,0 +1 @@
+pub mod helpers;

crates/tests-database/tests/db_story.rs

@@ -0,0 +1,213 @@
+use atuin_common::{
+    record::{EncryptedData, Host, HostId, Record, RecordIdx},
+    utils::{crypto_random_string, uuid_v7},
+};
+use atuin_server_database::{
+    Database, DbSettings, DbType,
+    models::{NewHistory, NewSession, NewUser, User},
+};
+use atuin_server_postgres::Postgres;
+use atuin_server_sqlite::Sqlite;
+use tests_database::helpers::{create_test_db, destroy_test_db};
+use time::OffsetDateTime;
+use uuid::Uuid;
+
+struct TestDb {
+    settings: DbSettings,
+}
+
+impl TestDb {
+    async fn new() -> eyre::Result<Self> {
+        let settings = create_test_db().await?;
+        Ok(Self { settings })
+    }
+}
+
+impl Drop for TestDb {
+    fn drop(&mut self) {
+        let settings = self.settings.clone();
+        let _ = std::thread::spawn(move || {
+            let rt = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .unwrap();
+            rt.block_on(async {
+                if let Err(e) = destroy_test_db(&settings).await {
+                    eprintln!("Failed to destroy test db: {:?}", e);
+                }
+            });
+        })
+        .join();
+    }
+}
+
+/// This test runs through a story of using the database. The goal is to fully exercise all DB code
+/// in a single repeatable manner.
+#[tokio::test]
+async fn test_full_db_story() -> eyre::Result<()> {
+    let test_db = TestDb::new().await?;
+    let settings = &test_db.settings;
+
+    match settings.db_type() {
+        DbType::Postgres => run_the_test::<Postgres>(settings).await,
+        DbType::Sqlite => run_the_test::<Sqlite>(settings).await,
+        DbType::Unknown => todo!(),
+    }
+}
+
+async fn run_the_test<DB: Database>(settings: &DbSettings) -> eyre::Result<()> {
+    let db = DB::new(settings).await?;
+    // register a user
+    let new_user = NewUser {
+        username: "foo".to_owned(),
+        email: "foo@example.com".to_owned(),
+        password: "hunter2".to_owned(),
+    };
+    let user_id = db.add_user(&new_user).await?;
+    assert_ne!(user_id, 0);
+
+    let token = crypto_random_string::<24>();
+    let new_session = NewSession {
+        user_id,
+        token: token.clone(),
+    };
+    db.add_session(&new_session).await?;
+
+    // The user is now registered and has a session. This happens when a user logs in
+    let user = db.get_session_user(&token).await?;
+    assert_eq!(user.username, "foo");
+
+    let session = db.get_session(&token).await?;
+    assert_eq!(session.user_id, user_id);
+
+    let user = db.get_user("foo").await?;
+    assert_eq!(user.password, "hunter2");
+
+    // Lets change the password
+    let user = User {
+        email: "foo@example.com".to_owned(),
+        id: user_id,
+        password: "hunter3".to_owned(),
+        username: "foo".to_owned(),
+    };
+    db.update_user_password(&user).await?;
+
+    let user = db.get_user("foo").await?;
+    assert_eq!(user.password, "hunter3");
+
+    // add some history
+    let h = vec![
+        generate_history(user_id),
+        generate_history(user_id),
+        generate_history(user_id),
+        generate_history(user_id),
+    ];
+    db.add_history(&h).await?;
+
+    assert_eq!(db.count_history(&user).await?, 4);
+
+    // AFAICT history is not used any more so I'm not going figure out how to take the timestamps
+    // from generated history into this
+    // assert_eq!(db.count_history_range(&user).await?, 4);
+
+    db.delete_history(&user, h[0].client_id.clone()).await?;
+    let deleted_history = db.deleted_history(&user).await?;
+    assert_eq!(deleted_history.len(), 1);
+
+    // add a bunch of records
+    let host_a = Host::new(HostId(uuid_v7()));
+    let host_b = Host::new(HostId(uuid_v7()));
+    let records = vec![
+        generate_record(&host_a, 1),
+        generate_record(&host_b, 2),
+        generate_record(&host_a, 2),
+        generate_record(&host_b, 2),
+        generate_record(&host_a, 3),
+        generate_record(&host_a, 4),
+        generate_record(&host_a, 5),
+        generate_record(&host_a, 6),
+    ];
+    db.add_records(&user, &records).await?;
+
+    let status = db.status(&user).await?;
+    assert!(status.hosts.contains_key(&host_a.id));
+    assert!(status.hosts.contains_key(&host_b.id));
+    assert_eq!(
+        status
+            .hosts
+            .get(&host_a.id)
+            .unwrap()
+            .get("history")
+            .unwrap()
+            .clone(),
+        6
+    );
+    assert_eq!(
+        status
+            .hosts
+            .get(&host_b.id)
+            .unwrap()
+            .get("history")
+            .unwrap()
+            .clone(),
+        2
+    );
+
+    // Get 3 records from the beginning
+    let recs = db
+        .next_records(&user, host_a.id, "history".into(), None, 3)
+        .await?;
+    assert_eq!(recs.len(), 3);
+    assert_eq!(recs[0].idx, 1);
+    assert_eq!(recs.last().unwrap().idx, 3);
+
+    // Get from the end, for host a. Get more than exists
+    let recs = db
+        .next_records(&user, host_a.id, "history".into(), Some(4), 10)
+        .await?;
+    assert_eq!(recs.len(), 3);
+    assert_eq!(recs[0].idx, 4); // check the head record is idx 4
+    assert_eq!(recs.last().unwrap().idx, 6);
+
+    // delete_store
+    db.delete_store(&user).await?;
+    let recs = db
+        .next_records(&user, host_a.id, "history".into(), Some(4), 10)
+        .await?;
+    assert_eq!(recs.len(), 0);
+
+    Ok(())
+}
+
+fn generate_history(user_id: i64) -> NewHistory {
+    use fake::Fake;
+    use fake::faker::lorem::en::*;
+
+    let data: String = Sentence(1..3).fake();
+    let hostname: String = "foo".to_owned();
+    let client_id = Uuid::new_v4().to_string();
+
+    let timestamp: OffsetDateTime = OffsetDateTime::now_utc();
+
+    NewHistory {
+        client_id,
+        user_id,
+        hostname,
+        timestamp,
+        data,
+    }
+}
+
+fn generate_record(host: &Host, idx: RecordIdx) -> Record<EncryptedData> {
+    let data = EncryptedData {
+        data: "some data".into(),
+        content_encryption_key: "key".into(),
+    };
+    Record::builder()
+        .idx(idx)
+        .host(host.clone())
+        .version("2".into())
+        .tag("history".into())
+        .data(data)
+        .build()
+}