From 7a172c6fdb0b946a00da0543e86cc6a38abe1e62 Mon Sep 17 00:00:00 2001 From: Jakob Date: Thu, 2 Apr 2026 08:15:05 +0200 Subject: [PATCH] Simplify database schema: remove card_type, card_id, add card_number Refactor the database schema to better model the data relationships: Schema changes: - Removed cards.card_type (redundant, identical to card_number) - Removed transactions.card_id (unnecessary indirection) - Added transactions.card_number (stores card number for all transactions) - Made cards.customer_id NOT NULL (every card must belong to a customer) - Made transactions.customer_id nullable (NULL for anonymized transactions) Import logic changes: - Only create cards for known customers (transactions with customer_number) - Store card_number for ALL transactions (including anonymized) - Skip cards/customer creation for anonymized transactions Additional changes: - Add 'db reset' command to drop and recreate database - Update migration file with new schema This simplifies queries and better reflects the data model: - Cards table: authoritative mapping of card_number -> customer_id - Transactions table: stores all raw data including anonymized cards - Customer relationship via JOIN on card_number for known customers --- migrations/002_schema.sql | 13 +++----- src/commands/db.rs | 43 ++++++++++++++++++++----- src/commands/import.rs | 66 ++++++++++++++++++--------------------- src/commands/mod.rs | 2 +- src/db/models.rs | 10 +++--- src/db/repository.rs | 20 ++++++------ src/main.rs | 10 ++++-- 7 files changed, 93 insertions(+), 71 deletions(-) diff --git a/migrations/002_schema.sql b/migrations/002_schema.sql index 9c07765..4025b84 100644 --- a/migrations/002_schema.sql +++ b/migrations/002_schema.sql @@ -13,13 +13,12 @@ CREATE TABLE IF NOT EXISTS customers ( CREATE TABLE IF NOT EXISTS cards ( id INT UNSIGNED AUTO_INCREMENT PRIMARY KEY, card_number VARCHAR(50) NOT NULL UNIQUE, - card_type VARCHAR(50), - customer_id INT UNSIGNED NULL, + customer_id INT UNSIGNED NOT NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, INDEX idx_card_number (card_number), INDEX idx_customer_id (customer_id), - FOREIGN KEY (customer_id) REFERENCES customers(id) ON DELETE SET NULL + FOREIGN KEY (customer_id) REFERENCES customers(id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; CREATE TABLE IF NOT EXISTS transactions ( @@ -31,19 +30,17 @@ CREATE TABLE IF NOT EXISTS transactions ( price DECIMAL(8,4) NOT NULL, quality_code INT NOT NULL, quality_name VARCHAR(50) NOT NULL, + card_number VARCHAR(50) NOT NULL, station VARCHAR(20) NOT NULL, terminal VARCHAR(10) NOT NULL, pump VARCHAR(10) NOT NULL, receipt VARCHAR(20) NOT NULL, control_number VARCHAR(20), - card_id INT UNSIGNED NOT NULL, customer_id INT UNSIGNED NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, INDEX idx_transaction_date (transaction_date), INDEX idx_batch_number (batch_number), - INDEX idx_card_id (card_id), INDEX idx_customer_id (customer_id), - INDEX idx_station (station), - FOREIGN KEY (card_id) REFERENCES cards(id), - FOREIGN KEY (customer_id) REFERENCES customers(id) ON DELETE SET NULL + INDEX idx_card_number (card_number), + INDEX idx_station (station) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/src/commands/db.rs b/src/commands/db.rs index 3adb2fc..61c8d61 100644 --- a/src/commands/db.rs +++ b/src/commands/db.rs @@ -47,13 +47,12 @@ pub async fn run_db_setup(repo: &Repository, config: &Config) -> anyhow::Result< CREATE TABLE IF NOT EXISTS cards ( id INT UNSIGNED AUTO_INCREMENT PRIMARY KEY, card_number VARCHAR(50) NOT NULL UNIQUE, - card_type VARCHAR(50), - customer_id INT UNSIGNED NULL, + customer_id INT UNSIGNED NOT NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, INDEX idx_card_number (card_number), INDEX idx_customer_id (customer_id), - FOREIGN KEY (customer_id) REFERENCES customers(id) ON DELETE SET NULL + FOREIGN KEY (customer_id) REFERENCES customers(id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci "#, ) @@ -71,21 +70,19 @@ pub async fn run_db_setup(repo: &Repository, config: &Config) -> anyhow::Result< price DECIMAL(8,4) NOT NULL, quality_code INT NOT NULL, quality_name VARCHAR(50) NOT NULL, + card_number VARCHAR(50) NOT NULL, station VARCHAR(20) NOT NULL, terminal VARCHAR(10) NOT NULL, pump VARCHAR(10) NOT NULL, receipt VARCHAR(20) NOT NULL, control_number VARCHAR(20), - card_id INT UNSIGNED NOT NULL, customer_id INT UNSIGNED NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, INDEX idx_transaction_date (transaction_date), INDEX idx_batch_number (batch_number), - INDEX idx_card_id (card_id), INDEX idx_customer_id (customer_id), - INDEX idx_station (station), - FOREIGN KEY (card_id) REFERENCES cards(id), - FOREIGN KEY (customer_id) REFERENCES customers(id) ON DELETE SET NULL + INDEX idx_card_number (card_number), + INDEX idx_station (station) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci "#, ) @@ -97,3 +94,33 @@ pub async fn run_db_setup(repo: &Repository, config: &Config) -> anyhow::Result< Ok(()) } + +pub async fn run_db_reset(config: &Config) -> anyhow::Result<()> { + let env = &config.env; + println!("Resetting database for environment: {}", env.as_str()); + println!("Database: {}", env.database_name()); + + let database_url = &config.database.connection_url(); + let base_url = database_url.trim_end_matches(env.database_name()); + + let setup_pool = MySqlPoolOptions::new() + .max_connections(1) + .connect(base_url) + .await?; + + println!("Dropping database if exists..."); + sqlx::query(&format!("DROP DATABASE IF EXISTS {}", env.database_name())) + .execute(&setup_pool) + .await?; + + println!("Creating database..."); + sqlx::query(&format!("CREATE DATABASE {}", env.database_name())) + .execute(&setup_pool) + .await?; + + drop(setup_pool); + + println!("Database '{}' reset complete!", env.database_name()); + + Ok(()) +} diff --git a/src/commands/import.rs b/src/commands/import.rs index a24d11c..7c96dab 100644 --- a/src/commands/import.rs +++ b/src/commands/import.rs @@ -7,7 +7,7 @@ use std::fs::File; use std::path::Path; pub async fn run_import(csv_path: &Path, repo: &Repository) -> anyhow::Result<()> { - println!("Läser CSV-fil: {:?}", csv_path); + println!("Reading CSV file: {:?}", csv_path); let file = File::open(csv_path)?; let mut rdr = ReaderBuilder::new() @@ -18,27 +18,29 @@ pub async fn run_import(csv_path: &Path, repo: &Repository) -> anyhow::Result<() let mut transactions = Vec::new(); let mut seen_customers: HashMap = HashMap::new(); - let mut seen_cards: HashMap, Option)> = HashMap::new(); + let mut seen_cards: HashMap = HashMap::new(); for result in rdr.records() { let record = result?; if let Some(tx) = parse_record(&record)? { - let card_report_group: u8 = tx.card_report_group_number.parse().unwrap_or(0); - if !seen_customers.contains_key(&tx.customer_number) && !tx.customer_number.is_empty() { - seen_customers.insert(tx.customer_number.clone(), card_report_group); - } - if !seen_cards.contains_key(&tx.card_number) { - seen_cards.insert(tx.card_number.clone(), (Some(tx.card_type.clone()), None)); + if !tx.customer_number.is_empty() { + let card_report_group: u8 = tx.card_report_group_number.parse().unwrap_or(0); + if !seen_customers.contains_key(&tx.customer_number) { + seen_customers.insert(tx.customer_number.clone(), card_report_group); + } + if !seen_cards.contains_key(&tx.card_number) { + seen_cards.insert(tx.card_number.clone(), tx.customer_number.clone()); + } } transactions.push(tx); } } - println!("Hittade {} transaktioner", transactions.len()); - println!("Unika kunder: {}", seen_customers.len()); - println!("Unika kort: {}", seen_cards.len()); + println!("Found {} transactions", transactions.len()); + println!("Unique customers: {}", seen_customers.len()); + println!("Unique known cards: {}", seen_cards.len()); - println!("\nImporterar kunder..."); + println!("\nImporting customers..."); let mut customer_ids: HashMap = HashMap::new(); for (customer_number, card_report_group) in &seen_customers { let new_customer = NewCustomer { @@ -47,30 +49,29 @@ pub async fn run_import(csv_path: &Path, repo: &Repository) -> anyhow::Result<() }; let id = repo.upsert_customer(&new_customer).await?; customer_ids.insert(customer_number.clone(), id); - println!(" Kund {} -> id {}", customer_number, id); + println!(" Customer {} -> id {}", customer_number, id); } - println!("\nImporterar kort..."); + println!("\nImporting cards..."); let mut card_ids: HashMap = HashMap::new(); - for (card_number, (card_type, _)) in &mut seen_cards { - let customer_id = customer_ids.get(card_number).copied(); - let new_card = NewCard { - card_number: card_number.clone(), - card_type: card_type.clone(), - customer_id, - }; - let id = repo.upsert_card(&new_card).await?; - card_ids.insert(card_number.clone(), id); - *card_type = None; + for (card_number, customer_number) in &seen_cards { + if let Some(&customer_id) = customer_ids.get(customer_number) { + let new_card = NewCard { + card_number: card_number.clone(), + customer_id, + }; + let id = repo.upsert_card(&new_card).await?; + card_ids.insert(card_number.clone(), id); + println!(" Card {} -> customer {} -> id {}", card_number, customer_number, id); + } } - println!("\nImporterar transaktioner..."); + println!("\nImporting transactions..."); let batch_size = 500; let mut total_inserted = 0u64; let mut batch: Vec = Vec::with_capacity(batch_size); for tx in transactions { - let card_id = *card_ids.get(&tx.card_number).unwrap_or(&0); let customer_id = customer_ids.get(&tx.customer_number).copied(); let new_tx = NewTransaction { @@ -81,12 +82,12 @@ pub async fn run_import(csv_path: &Path, repo: &Repository) -> anyhow::Result<() price: tx.price, quality_code: tx.quality, quality_name: tx.quality_name, + card_number: tx.card_number, station: tx.station, terminal: tx.terminal, pump: tx.pump, receipt: tx.receipt, control_number: if tx.control_number.is_empty() { None } else { Some(tx.control_number) }, - card_id, customer_id, }; @@ -95,7 +96,7 @@ pub async fn run_import(csv_path: &Path, repo: &Repository) -> anyhow::Result<() if batch.len() >= batch_size { let inserted = repo.insert_transactions_batch(&batch).await?; total_inserted += inserted; - println!(" Inlagda {} transaktioner (totalt: {})", inserted, total_inserted); + println!(" Inserted {} transactions (total: {})", inserted, total_inserted); batch.clear(); } } @@ -103,10 +104,10 @@ pub async fn run_import(csv_path: &Path, repo: &Repository) -> anyhow::Result<() if !batch.is_empty() { let inserted = repo.insert_transactions_batch(&batch).await?; total_inserted += inserted; - println!(" Inlagda {} transaktioner (totalt: {})", inserted, total_inserted); + println!(" Inserted {} transactions (total: {})", inserted, total_inserted); } - println!("\nKlart!Importerade {} transaktioner", total_inserted); + println!("\nDone! Imported {} transactions", total_inserted); Ok(()) } @@ -120,7 +121,6 @@ struct CsvTransaction { quality: i32, quality_name: String, card_number: String, - card_type: String, customer_number: String, station: String, terminal: String, @@ -147,9 +147,6 @@ fn parse_record(record: &csv::StringRecord) -> anyhow::Result anyhow::Result, - pub customer_id: Option, + pub customer_id: u32, pub created_at: DateTime, pub updated_at: DateTime, } @@ -31,8 +30,7 @@ pub struct Card { #[derive(Debug, Clone)] pub struct NewCard { pub card_number: String, - pub card_type: Option, - pub customer_id: Option, + pub customer_id: u32, } #[derive(Debug, Clone, Serialize, Deserialize, FromRow)] @@ -45,12 +43,12 @@ pub struct Transaction { pub price: BigDecimal, pub quality_code: i32, pub quality_name: String, + pub card_number: String, pub station: String, pub terminal: String, pub pump: String, pub receipt: String, pub control_number: Option, - pub card_id: u32, pub customer_id: Option, pub created_at: DateTime, } @@ -64,11 +62,11 @@ pub struct NewTransaction { pub price: f64, pub quality_code: i32, pub quality_name: String, + pub card_number: String, pub station: String, pub terminal: String, pub pump: String, pub receipt: String, pub control_number: Option, - pub card_id: u32, pub customer_id: Option, } diff --git a/src/db/repository.rs b/src/db/repository.rs index 273b13c..b6060dd 100644 --- a/src/db/repository.rs +++ b/src/db/repository.rs @@ -59,16 +59,14 @@ impl Repository { pub async fn upsert_card(&self, card: &NewCard) -> anyhow::Result { sqlx::query( r#" - INSERT INTO cards (card_number, card_type, customer_id) - VALUES (?, ?, ?) + INSERT INTO cards (card_number, customer_id) + VALUES (?, ?) ON DUPLICATE KEY UPDATE - card_type = COALESCE(VALUES(card_type), card_type), - customer_id = COALESCE(VALUES(customer_id), customer_id), + customer_id = VALUES(customer_id), updated_at = CURRENT_TIMESTAMP "#, ) .bind(&card.card_number) - .bind(&card.card_type) .bind(card.customer_id) .execute(&self.pool) .await?; @@ -85,7 +83,7 @@ impl Repository { pub async fn find_card_by_number(&self, card_number: &str) -> anyhow::Result> { let result = sqlx::query_as( - "SELECT id, card_number, card_type, customer_id, created_at, updated_at + "SELECT id, card_number, customer_id, created_at, updated_at FROM cards WHERE card_number = ?", ) @@ -105,13 +103,13 @@ impl Repository { } let mut query = String::from( - "INSERT INTO transactions (transaction_date, batch_number, amount, volume, price, quality_code, quality_name, station, terminal, pump, receipt, control_number, card_id, customer_id) VALUES ", + "INSERT INTO transactions (transaction_date, batch_number, amount, volume, price, quality_code, quality_name, card_number, station, terminal, pump, receipt, control_number, customer_id) VALUES ", ); let mut values = Vec::new(); for tx in transactions { values.push(format!( - "('{}', '{}', {}, {}, {}, {}, '{}', '{}', '{}', '{}', '{}', {}, {}, {})", + "('{}', '{}', {}, {}, {}, {}, '{}', '{}', '{}', '{}', '{}', '{}', {}, {})", tx.transaction_date.format("%Y-%m-%d %H:%M:%S"), tx.batch_number, tx.amount, @@ -119,12 +117,12 @@ impl Repository { tx.price, tx.quality_code, tx.quality_name.replace("'", "''"), + tx.card_number.replace("'", "''"), tx.station, tx.terminal, tx.pump, tx.receipt, tx.control_number.as_ref().map(|s| format!("'{}'", s.replace("'", "''"))).unwrap_or_else(|| "NULL".to_string()), - tx.card_id, tx.customer_id.map(|id| id.to_string()).unwrap_or_else(|| "NULL".to_string()), )); } @@ -145,8 +143,8 @@ impl Repository { let result = sqlx::query_as( r#" SELECT t.id, t.transaction_date, t.batch_number, t.amount, t.volume, t.price, - t.quality_code, t.quality_name, t.station, t.terminal, t.pump, - t.receipt, t.control_number, t.card_id, t.customer_id, t.created_at + t.quality_code, t.quality_name, t.card_number, t.station, t.terminal, + t.pump, t.receipt, t.control_number, t.customer_id, t.created_at FROM transactions t JOIN customers c ON t.customer_id = c.id WHERE c.customer_number = ? diff --git a/src/main.rs b/src/main.rs index 6ac0128..bee48bd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -391,22 +391,27 @@ async fn main() -> anyhow::Result<()> { eprintln!("Usage: {} db [--env ]", clean_args[0]); eprintln!("Subcommands:"); eprintln!(" setup Create database and schema"); + eprintln!(" reset Drop and recreate database"); std::process::exit(1); } println!("Environment: {}", env.as_str()); let config = Config::load(env)?; - let pool = create_pool(&config.database.connection_url()).await?; - let repo = Repository::new(pool); match clean_args[2].as_str() { "setup" => { + let pool = create_pool(&config.database.connection_url()).await?; + let repo = Repository::new(pool); commands::run_db_setup(&repo, &config).await?; } + "reset" => { + commands::run_db_reset(&config).await?; + } _ => { eprintln!("Unknown db subcommand: {}", clean_args[2]); eprintln!("Subcommands:"); eprintln!(" setup Create database and schema"); + eprintln!(" reset Drop and recreate database"); std::process::exit(1); } } @@ -431,6 +436,7 @@ fn print_usage(program: &str) { eprintln!(" import [--env ] Import CSV data to database (default: prod)"); eprintln!(" generate Generate HTML invoices from CSV"); eprintln!(" db setup [--env ] Create database and schema (default: prod)"); + eprintln!(" db reset [--env ] Drop and recreate database (default: prod)"); eprintln!(" help Show this help message"); eprintln!(); eprintln!("Environments: prod (default), dev, test");