From 05ee74eb7b70f1534f0e753bdba2164a00058d2d Mon Sep 17 00:00:00 2001 From: antifallobst Date: Sun, 12 Nov 2023 20:49:04 +0100 Subject: [PATCH] feat: implemented the /backup/create [POST] endpoint and the 'NginxConfig' backup configuration --- .env | 4 +- .gitignore | 3 +- Cargo.lock | 147 ++++++++++++++++++++++++++++++- Cargo.toml | 3 + src/api/calls.rs | 30 +++---- src/api/handlers.rs | 27 ++++-- src/api/mod.rs | 13 ++- src/backend/backup/mod.rs | 116 +++++++++++++++++++++++- src/backend/backup/preset.rs | 4 + src/backend/backup/worker/mod.rs | 126 ++++++++++++++++++++++++++ src/backend/mod.rs | 19 ++++ src/main.rs | 5 +- 12 files changed, 466 insertions(+), 31 deletions(-) create mode 100644 src/backend/backup/worker/mod.rs diff --git a/.env b/.env index ce5aa37..4b7276d 100644 --- a/.env +++ b/.env @@ -1 +1,3 @@ -DATABASE_URL="sqlite:store.db" \ No newline at end of file +DATABASE_URL="sqlite:store.db" +NC_AW_BACKUP_PATH=test +NC_AW_HOST_PATH=/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index 76dfc75..1dfefcd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea /target -store.db \ No newline at end of file +store.db +test/ \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 895c1a9..1e7fd04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -55,7 +55,7 @@ dependencies = [ "tokio", "tokio-util", "tracing", - "zstd", + "zstd 0.12.4", ] [[package]] @@ -218,12 +218,26 @@ dependencies = [ "actix-web", "actix-web-httpauth", "anyhow", + "chrono", "env_logger", "log", "serde", "serde_json", "sqlx", "tokio", + "walkdir", + "zip", +] + +[[package]] +name = "aes" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", ] [[package]] @@ -404,6 +418,27 @@ dependencies = [ "bytes", ] +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cc" version = "1.0.83" @@ -428,16 +463,34 @@ checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" dependencies = [ "android-tzdata", "iana-time-zone", + "js-sys", "num-traits", + "wasm-bindgen", "windows-targets", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "const-oid" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "convert_case" version = "0.4.0" @@ -959,6 +1012,15 @@ dependencies = [ "hashbrown 0.14.2", ] +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + [[package]] name = "is-terminal" version = "0.4.9" @@ -1235,12 +1297,35 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core", + "subtle", +] + [[package]] name = "paste" version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest", + "hmac", + "password-hash", + "sha2", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -1447,6 +1532,15 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2077,6 +2171,16 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2275,13 +2379,52 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "aes", + "byteorder", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "flate2", + "hmac", + "pbkdf2", + "sha1", + "time", + "zstd 0.11.2+zstd.1.5.2", +] + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe 5.0.2+zstd.1.5.2", +] + [[package]] name = "zstd" version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" dependencies = [ - "zstd-safe", + "zstd-safe 6.0.6", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b7bfb31..82494b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,9 @@ serde = { version = "1.0.183", features = ["derive"] } serde_json = "1.0.108" env_logger = "0.10" log = "0.4" +chrono = "0.4.31" +zip = "0.6.6" +walkdir = "2.4.0" sqlx = { version = "0.7.1", features = ["runtime-tokio", "sqlite", "chrono"] } actix-web = "4" actix-web-httpauth = "0.8.0" diff --git a/src/api/calls.rs b/src/api/calls.rs index bf0dab9..016442a 100644 --- a/src/api/calls.rs +++ b/src/api/calls.rs @@ -3,21 +3,21 @@ use actix_web::{delete, get, post, web, HttpResponse, Responder}; use actix_web_httpauth::extractors::bearer::BearerAuth; use log::error; -// #[post("/backup/create")] -// async fn backup_create( -// data: web::Data, -// body: web::Json, -// ) -> impl Responder { -// match handlers::backup_create(&data.pool, body.into_inner()).await { -// Ok(resp) => match resp { -// BackupCreateResponse::Success => HttpResponse::Ok().finish(), -// }, -// Err(e) => { -// error!("While handling /backup/create [POST] request: {e}"); -// HttpResponse::InternalServerError().finish() -// } -// } -// } +#[post("/backup/create")] +async fn backup_create( + data: web::Data, + body: web::Json, +) -> impl Responder { + match handlers::backup_create(&data.pool, data.backup_tx.clone(), body.into_inner()).await { + Ok(resp) => match resp { + BackupCreateResponse::Success => HttpResponse::Ok().finish(), + }, + Err(e) => { + error!("While handling /backup/create [POST] request: {e}"); + HttpResponse::InternalServerError().finish() + } + } +} #[post("/backup/preset")] async fn backup_preset_post( diff --git a/src/api/handlers.rs b/src/api/handlers.rs index c58d80a..ed77fd5 100644 --- a/src/api/handlers.rs +++ b/src/api/handlers.rs @@ -1,15 +1,28 @@ use crate::api::data::*; use crate::backend::backup; -use anyhow::Result; +use anyhow::{Context, Result}; use log::warn; use sqlx::sqlite::SqlitePool; +use tokio::sync::mpsc; -// pub async fn backup_create( -// pool: &SqlitePool, -// request: BackupCreateRequest, -// ) -> Result { -// Ok(BackupCreateResponse::Success) -// } +pub async fn backup_create( + pool: &SqlitePool, + tx: mpsc::Sender, + request: BackupCreateRequest, +) -> Result { + let config = match request { + BackupCreateRequest::Preset(preset) => backup::preset::Preset::load(pool, &preset) + .await? + .context(format!("Failed to look up preset '{preset}' !"))? + .config() + .to_owned(), + BackupCreateRequest::Config(cfg) => cfg.into(), + }; + + let backup = backup::Backup::new(pool, tx, config).await?; + + Ok(BackupCreateResponse::Success) +} pub async fn backup_preset_post( pool: &SqlitePool, diff --git a/src/api/mod.rs b/src/api/mod.rs index c61b95b..7fa3230 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -2,19 +2,27 @@ pub mod calls; pub mod data; mod handlers; +use crate::backend::backup; use actix_web::{web, App, HttpServer}; use anyhow::Result; use sqlx::sqlite::SqlitePool; +use tokio::sync::mpsc; pub struct State { pool: SqlitePool, token: String, + backup_tx: mpsc::Sender, } -pub async fn start(port: u16, pool: SqlitePool, token: String) -> Result<()> { +pub async fn start( + port: u16, + pool: SqlitePool, + token: String, + backup_tx: mpsc::Sender, +) -> Result<()> { let _ = HttpServer::new(move || { App::new() - // .service(calls::backup_create) + .service(calls::backup_create) .service(calls::backup_preset_post) .service(calls::backup_preset_get) .service(calls::backup_preset_id_get) @@ -22,6 +30,7 @@ pub async fn start(port: u16, pool: SqlitePool, token: String) -> Result<()> { .app_data(web::Data::new(State { pool: pool.clone(), token: token.to_owned(), + backup_tx: backup_tx.to_owned(), })) }) .bind(("0.0.0.0", port))? diff --git a/src/backend/backup/mod.rs b/src/backend/backup/mod.rs index 821865a..68ace17 100644 --- a/src/backend/backup/mod.rs +++ b/src/backend/backup/mod.rs @@ -1,12 +1,18 @@ pub mod preset; +pub mod worker; use crate::api; +use anyhow::{Context, Error, Result}; +use chrono::NaiveDateTime; use serde::{Deserialize, Serialize}; +use sqlx::SqlitePool; +use std::path::PathBuf; +use tokio::sync::mpsc; -#[derive(Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone)] struct DockerConfig {} -#[derive(Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct Config { nginx_config: bool, mail_server: bool, @@ -52,3 +58,109 @@ impl From for api::data::BackupConfig { } } } + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub enum Status { + Pending, + Processing, + Finished, + Failed, +} + +pub struct RawBackup { + id: i64, + time: i64, + config: String, + path: String, + status: String, +} + +#[derive(Debug, Clone)] +pub struct Backup { + id: u64, + time: NaiveDateTime, + config: Config, + path: PathBuf, + status: Status, +} + +impl TryFrom for Backup { + type Error = Error; + + fn try_from(value: RawBackup) -> std::result::Result { + Ok(Self { + id: value.id as u64, + time: NaiveDateTime::from_timestamp_opt(value.time, 0) + .context("Failed to convert timestamp into NaiveTime")?, + config: serde_json::from_str(&value.config)?, + path: value.path.into(), + status: serde_json::from_str(&value.status)?, + }) + } +} + +impl Backup { + pub async fn new(pool: &SqlitePool, tx: mpsc::Sender, config: Config) -> Result { + let time = chrono::Utc::now().naive_utc(); + let timestamp = time.timestamp(); + let config = serde_json::to_string(&config)?; + let status = serde_json::to_string(&Status::Pending)?; + + let id = sqlx::query!( + r#"INSERT INTO Backups (time, config, path, status) VALUES($1, $2, $3, $4) RETURNING id;"#, + timestamp, + config, + "", + status + ) + .fetch_one(pool) + .await? + .id; + + let path = format!( + "{}/{}.bak", + env!("NC_AW_BACKUP_PATH"), + time.format("%H-%M-%S_%d-%m-%Y") + ); + + sqlx::query!(r#"UPDATE Backups SET path = $1 WHERE id = $2;"#, path, id) + .execute(pool) + .await?; + + let backup = Self::load(pool, id) + .await? + .context("The just created backup record can't be found in the database!")?; + + tx.send(backup.clone()).await?; + + Ok(backup) + } + + pub async fn load(pool: &SqlitePool, id: i64) -> Result> { + let query_result = + sqlx::query_as!(RawBackup, r#"SELECT * FROM Backups WHERE id = $1;"#, id) + .fetch_one(pool) + .await; + + match query_result { + Ok(raw) => Ok(Some(raw.try_into()?)), + Err(sqlx::Error::RowNotFound) => Ok(None), + Err(e) => Err(Error::new(e)), + } + } + + pub async fn set_status(&self, pool: &SqlitePool, status: Status) -> Result<()> { + let status = serde_json::to_string(&status)?; + let id = self.id as i64; + + sqlx::query!( + r#"UPDATE Backups SET status = $1 WHERE id = $2;"#, + status, + id + ) + .execute(pool) + .await?; + + Ok(()) + } +} diff --git a/src/backend/backup/preset.rs b/src/backend/backup/preset.rs index f6fd574..ec47910 100644 --- a/src/backend/backup/preset.rs +++ b/src/backend/backup/preset.rs @@ -87,6 +87,10 @@ impl Preset { } } + pub fn config(&self) -> &backup::Config { + &self.config + } + pub async fn delete(&self, pool: &SqlitePool) -> Result<()> { sqlx::query!(r#"DELETE FROM Presets WHERE id = $1;"#, self.id) .execute(pool) diff --git a/src/backend/backup/worker/mod.rs b/src/backend/backup/worker/mod.rs new file mode 100644 index 0000000..cdda721 --- /dev/null +++ b/src/backend/backup/worker/mod.rs @@ -0,0 +1,126 @@ +use crate::backend::backup; +use anyhow::{bail, Context, Result}; +use log::{error, info}; +use sqlx::SqlitePool; +use std::fs::File; +use std::io::{copy, BufReader, Write}; +use std::path::Path; +use tokio::sync::mpsc; +use walkdir::{DirEntry, WalkDir}; +use zip::{write::FileOptions, CompressionMethod, ZipWriter}; + +fn add_dir_to_archive( + archive: &mut ZipWriter, + dir: &Path, + options: FileOptions, +) -> Result<()> { + for entry in WalkDir::new(dir) + .follow_links(true) + .into_iter() + .filter_map(|e| e.ok()) + { + let fname = entry.file_name(); + let ftype = entry.file_type(); + + if ftype.is_dir() { + archive.add_directory(entry.path().to_string_lossy(), options)?; + } else if ftype.is_file() { + archive.start_file(entry.path().to_string_lossy(), options)?; + let mut reader = BufReader::new(File::open(entry.path())?); + copy(&mut reader, archive)?; + } + } + Ok(()) +} + +fn perform_backup(backup: backup::Backup, pool: SqlitePool) -> Result<()> { + let host = env!("NC_AW_HOST_PATH"); + let archive_file = File::create(&backup.path)?; + let mut archive = ZipWriter::new(archive_file); + let options = FileOptions::default() + .compression_method(CompressionMethod::Bzip2) + .compression_level(Some(9)); + + info!("Starting backup..."); + + if backup.config.nginx_config { + info!("Starting NGINX config backup..."); + add_dir_to_archive( + &mut archive, + &Path::new(&format!("{host}/etc/nginx")), + options, + )?; + } + + if backup.config.mail_server { + info!("Starting mail server backup..."); + bail!("The config option 'mail_server' is not implemented yet!"); + } + + if let Some(cfg) = &backup.config.docker { + info!("Starting docker backup..."); + bail!("The config option 'docker' is not implemented yet!"); + } + + archive.finish()?; + + info!("Finished backup."); + Ok(()) +} + +pub async fn start(pool: SqlitePool) -> Result> { + let (tx, mut rx) = mpsc::channel(128); + + tokio::task::spawn(async move { + loop { + let backup: backup::Backup = rx + .recv() + .await + .expect("Failed to read from the backup workers internal MPSC channel!"); + + let cloned_backup = backup.clone(); + + let result: Result<()> = async { + backup + .set_status(&pool, backup::Status::Processing) + .await + .expect("Failed to set backup status!"); + + let cloned_backup = backup.clone(); + let cloned_pool = pool.clone(); + match tokio::task::spawn_blocking(|| perform_backup(cloned_backup, cloned_pool)) + .await? + { + Ok(_) => (), + Err(e) => { + tokio::fs::remove_file(backup.path).await?; + return Err(e); + } + } + + backup + .set_status(&pool, backup::Status::Finished) + .await + .context("Failed to set backup status!")?; + Ok(()) + } + .await; + + match result { + Ok(_) => (), + Err(e) => { + match cloned_backup + .set_status(&pool, backup::Status::Failed) + .await + { + Ok(_) => (), + Err(e) => error!("Failed to set backup failure status: {e}"), + } + error!("The backup worker thread failed: {e}"); + } + } + } + }); + + Ok(tx) +} diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 112594f..47f9d35 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -16,5 +16,24 @@ pub async fn prepare(pool: &SqlitePool) -> Result<()> { ) .execute(pool) .await?; + + sqlx::query!( + r#" + CREATE TABLE IF NOT EXISTS Backups ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + time INTEGER NOT NULL, + config TEXT NOT NULL, + path TEXT NOT NULL, + status TEXT NOT NULL + ); + "# + ) + .execute(pool) + .await?; + + if !tokio::fs::try_exists(env!("NC_AW_BACKUP_PATH")).await? { + tokio::fs::create_dir(env!("NC_AW_BACKUP_PATH")).await?; + } + Ok(()) } diff --git a/src/main.rs b/src/main.rs index c96347a..e407603 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ mod api; mod backend; +use crate::backend::backup; use anyhow::Result; use log::info; use sqlx::SqlitePool; @@ -24,7 +25,9 @@ async fn main() -> Result<()> { .await?; backend::prepare(&pool).await?; - api::start(6969, pool, token.to_owned()).await?; + let backup_tx = backup::worker::start(pool.clone()).await?; + + api::start(6969, pool, token.to_owned(), backup_tx).await?; Ok(()) }