jakob/rusty-petroleum
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Simplify database schema: remove card_type, card_id, add card_number

Browse Source 
Refactor the database schema to better model the data relationships:

Schema changes:
- Removed cards.card_type (redundant, identical to card_number)
- Removed transactions.card_id (unnecessary indirection)
- Added transactions.card_number (stores card number for all transactions)
- Made cards.customer_id NOT NULL (every card must belong to a customer)
- Made transactions.customer_id nullable (NULL for anonymized transactions)

Import logic changes:
- Only create cards for known customers (transactions with customer_number)
- Store card_number for ALL transactions (including anonymized)
- Skip cards/customer creation for anonymized transactions

Additional changes:
- Add 'db reset' command to drop and recreate database
- Update migration file with new schema

This simplifies queries and better reflects the data model:
- Cards table: authoritative mapping of card_number -> customer_id
- Transactions table: stores all raw data including anonymized cards
- Customer relationship via JOIN on card_number for known customers
This commit is contained in:
Jakob
2026-04-02 08:15:05 +02:00
parent cd46368f79
commit 7a172c6fdb
7 changed files with 93 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
migrations/002_schema.sql
View File

@@ -13,13 +13,12 @@ CREATE TABLE IF NOT EXISTS customers (
CREATE TABLE IF NOT EXISTS cards (
id INT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
card_number VARCHAR(50) NOT NULL UNIQUE,
card_type VARCHAR(50),
customer_id INT UNSIGNED NULL,
customer_id INT UNSIGNED NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
INDEX idx_card_number (card_number),
INDEX idx_customer_id (customer_id),
FOREIGN KEY (customer_id) REFERENCES customers(id) ON DELETE SET NULL
FOREIGN KEY (customer_id) REFERENCES customers(id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE IF NOT EXISTS transactions (
@@ -31,19 +30,17 @@ CREATE TABLE IF NOT EXISTS transactions (
price DECIMAL(8,4) NOT NULL,
quality_code INT NOT NULL,
quality_name VARCHAR(50) NOT NULL,
card_number VARCHAR(50) NOT NULL,
station VARCHAR(20) NOT NULL,
terminal VARCHAR(10) NOT NULL,
pump VARCHAR(10) NOT NULL,
receipt VARCHAR(20) NOT NULL,
control_number VARCHAR(20),
card_id INT UNSIGNED NOT NULL,
customer_id INT UNSIGNED NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
INDEX idx_transaction_date (transaction_date),
INDEX idx_batch_number (batch_number),
INDEX idx_card_id (card_id),
INDEX idx_customer_id (customer_id),
INDEX idx_station (station),
FOREIGN KEY (card_id) REFERENCES cards(id),
FOREIGN KEY (customer_id) REFERENCES customers(id) ON DELETE SET NULL
INDEX idx_card_number (card_number),
INDEX idx_station (station)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

43
src/commands/db.rs
View File

@@ -47,13 +47,12 @@ pub async fn run_db_setup(repo: &Repository, config: &Config) -> anyhow::Result<
CREATE TABLE IF NOT EXISTS cards (
id INT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
card_number VARCHAR(50) NOT NULL UNIQUE,
card_type VARCHAR(50),
customer_id INT UNSIGNED NULL,
customer_id INT UNSIGNED NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
INDEX idx_card_number (card_number),
INDEX idx_customer_id (customer_id),
FOREIGN KEY (customer_id) REFERENCES customers(id) ON DELETE SET NULL
FOREIGN KEY (customer_id) REFERENCES customers(id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
"#,
)
@@ -71,21 +70,19 @@ pub async fn run_db_setup(repo: &Repository, config: &Config) -> anyhow::Result<
price DECIMAL(8,4) NOT NULL,
quality_code INT NOT NULL,
quality_name VARCHAR(50) NOT NULL,
card_number VARCHAR(50) NOT NULL,
station VARCHAR(20) NOT NULL,
terminal VARCHAR(10) NOT NULL,
pump VARCHAR(10) NOT NULL,
receipt VARCHAR(20) NOT NULL,
control_number VARCHAR(20),
card_id INT UNSIGNED NOT NULL,
customer_id INT UNSIGNED NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
INDEX idx_transaction_date (transaction_date),
INDEX idx_batch_number (batch_number),
INDEX idx_card_id (card_id),
INDEX idx_customer_id (customer_id),
INDEX idx_station (station),
FOREIGN KEY (card_id) REFERENCES cards(id),
FOREIGN KEY (customer_id) REFERENCES customers(id) ON DELETE SET NULL
INDEX idx_card_number (card_number),
INDEX idx_station (station)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
"#,
)
@@ -97,3 +94,33 @@ pub async fn run_db_setup(repo: &Repository, config: &Config) -> anyhow::Result<
Ok(())
}
pub async fn run_db_reset(config: &Config) -> anyhow::Result<()> {
let env = &config.env;
println!("Resetting database for environment: {}", env.as_str());
println!("Database: {}", env.database_name());
let database_url = &config.database.connection_url();
let base_url = database_url.trim_end_matches(env.database_name());
let setup_pool = MySqlPoolOptions::new()
.max_connections(1)
.connect(base_url)
.await?;
println!("Dropping database if exists...");
sqlx::query(&format!("DROP DATABASE IF EXISTS {}", env.database_name()))
.execute(&setup_pool)
.await?;
println!("Creating database...");
sqlx::query(&format!("CREATE DATABASE {}", env.database_name()))
.execute(&setup_pool)
.await?;
drop(setup_pool);
println!("Database '{}' reset complete!", env.database_name());
Ok(())
}

66
src/commands/import.rs
View File

@@ -7,7 +7,7 @@ use std::fs::File;
use std::path::Path;
pub async fn run_import(csv_path: &Path, repo: &Repository) -> anyhow::Result<()> {
println!("Läser CSV-fil: {:?}", csv_path);
println!("Reading CSV file: {:?}", csv_path);
let file = File::open(csv_path)?;
let mut rdr = ReaderBuilder::new()
@@ -18,27 +18,29 @@ pub async fn run_import(csv_path: &Path, repo: &Repository) -> anyhow::Result<()
let mut transactions = Vec::new();
let mut seen_customers: HashMap<String, u8> = HashMap::new();
let mut seen_cards: HashMap<String, (Option<String>, Option<u32>)> = HashMap::new();
let mut seen_cards: HashMap<String, String> = HashMap::new();
for result in rdr.records() {
let record = result?;
if let Some(tx) = parse_record(&record)? {
let card_report_group: u8 = tx.card_report_group_number.parse().unwrap_or(0);
if !seen_customers.contains_key(&tx.customer_number) && !tx.customer_number.is_empty() {
seen_customers.insert(tx.customer_number.clone(), card_report_group);
}
if !seen_cards.contains_key(&tx.card_number) {
seen_cards.insert(tx.card_number.clone(), (Some(tx.card_type.clone()), None));
if !tx.customer_number.is_empty() {
let card_report_group: u8 = tx.card_report_group_number.parse().unwrap_or(0);
if !seen_customers.contains_key(&tx.customer_number) {
seen_customers.insert(tx.customer_number.clone(), card_report_group);
}
if !seen_cards.contains_key(&tx.card_number) {
seen_cards.insert(tx.card_number.clone(), tx.customer_number.clone());
}
}
transactions.push(tx);
}
}
println!("Hittade {} transaktioner", transactions.len());
println!("Unika kunder: {}", seen_customers.len());
println!("Unika kort: {}", seen_cards.len());
println!("Found {} transactions", transactions.len());
println!("Unique customers: {}", seen_customers.len());
println!("Unique known cards: {}", seen_cards.len());
println!("\nImporterar kunder...");
println!("\nImporting customers...");
let mut customer_ids: HashMap<String, u32> = HashMap::new();
for (customer_number, card_report_group) in &seen_customers {
let new_customer = NewCustomer {
@@ -47,30 +49,29 @@ pub async fn run_import(csv_path: &Path, repo: &Repository) -> anyhow::Result<()
};
let id = repo.upsert_customer(&new_customer).await?;
customer_ids.insert(customer_number.clone(), id);
println!(" Kund {} -> id {}", customer_number, id);
println!(" Customer {} -> id {}", customer_number, id);
}
println!("\nImporterar kort...");
println!("\nImporting cards...");
let mut card_ids: HashMap<String, u32> = HashMap::new();
for (card_number, (card_type, _)) in &mut seen_cards {
let customer_id = customer_ids.get(card_number).copied();
let new_card = NewCard {
card_number: card_number.clone(),
card_type: card_type.clone(),
customer_id,
};
let id = repo.upsert_card(&new_card).await?;
card_ids.insert(card_number.clone(), id);
*card_type = None;
for (card_number, customer_number) in &seen_cards {
if let Some(&customer_id) = customer_ids.get(customer_number) {
let new_card = NewCard {
card_number: card_number.clone(),
customer_id,
};
let id = repo.upsert_card(&new_card).await?;
card_ids.insert(card_number.clone(), id);
println!(" Card {} -> customer {} -> id {}", card_number, customer_number, id);
}
}
println!("\nImporterar transaktioner...");
println!("\nImporting transactions...");
let batch_size = 500;
let mut total_inserted = 0u64;
let mut batch: Vec<NewTransaction> = Vec::with_capacity(batch_size);
for tx in transactions {
let card_id = *card_ids.get(&tx.card_number).unwrap_or(&0);
let customer_id = customer_ids.get(&tx.customer_number).copied();
let new_tx = NewTransaction {
@@ -81,12 +82,12 @@ pub async fn run_import(csv_path: &Path, repo: &Repository) -> anyhow::Result<()
price: tx.price,
quality_code: tx.quality,
quality_name: tx.quality_name,
card_number: tx.card_number,
station: tx.station,
terminal: tx.terminal,
pump: tx.pump,
receipt: tx.receipt,
control_number: if tx.control_number.is_empty() { None } else { Some(tx.control_number) },
card_id,
customer_id,
};
@@ -95,7 +96,7 @@ pub async fn run_import(csv_path: &Path, repo: &Repository) -> anyhow::Result<()
if batch.len() >= batch_size {
let inserted = repo.insert_transactions_batch(&batch).await?;
total_inserted += inserted;
println!(" Inlagda {} transaktioner (totalt: {})", inserted, total_inserted);
println!(" Inserted {} transactions (total: {})", inserted, total_inserted);
batch.clear();
}
}
@@ -103,10 +104,10 @@ pub async fn run_import(csv_path: &Path, repo: &Repository) -> anyhow::Result<()
if !batch.is_empty() {
let inserted = repo.insert_transactions_batch(&batch).await?;
total_inserted += inserted;
println!(" Inlagda {} transaktioner (totalt: {})", inserted, total_inserted);
println!(" Inserted {} transactions (total: {})", inserted, total_inserted);
}
println!("\nKlart!Importerade {} transaktioner", total_inserted);
println!("\nDone! Imported {} transactions", total_inserted);
Ok(())
}
@@ -120,7 +121,6 @@ struct CsvTransaction {
quality: i32,
quality_name: String,
card_number: String,
card_type: String,
customer_number: String,
station: String,
terminal: String,
@@ -147,9 +147,6 @@ fn parse_record(record: &csv::StringRecord) -> anyhow::Result<Option<CsvTransact
}
let customer_number = get_field(record, 9).to_string();
if customer_number.is_empty() {
return Ok(None);
}
Ok(Some(CsvTransaction {
date,
@@ -160,7 +157,6 @@ fn parse_record(record: &csv::StringRecord) -> anyhow::Result<Option<CsvTransact
quality: get_field(record, 5).parse().unwrap_or(0),
quality_name: get_field(record, 6).to_string(),
card_number: get_field(record, 7).to_string(),
card_type: get_field(record, 8).to_string(),
customer_number,
station: get_field(record, 10).to_string(),
terminal: get_field(record, 11).to_string(),

2
src/commands/mod.rs
View File

@@ -1,5 +1,5 @@
pub mod db;
pub mod import;
pub use db::run_db_setup;
pub use db::{run_db_reset, run_db_setup};
pub use import::run_import;

10
src/db/models.rs
View File

@@ -22,8 +22,7 @@ pub struct NewCustomer {
pub struct Card {
pub id: u32,
pub card_number: String,
pub card_type: Option<String>,
pub customer_id: Option<u32>,
pub customer_id: u32,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
@@ -31,8 +30,7 @@ pub struct Card {
#[derive(Debug, Clone)]
pub struct NewCard {
pub card_number: String,
pub card_type: Option<String>,
pub customer_id: Option<u32>,
pub customer_id: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
@@ -45,12 +43,12 @@ pub struct Transaction {
pub price: BigDecimal,
pub quality_code: i32,
pub quality_name: String,
pub card_number: String,
pub station: String,
pub terminal: String,
pub pump: String,
pub receipt: String,
pub control_number: Option<String>,
pub card_id: u32,
pub customer_id: Option<u32>,
pub created_at: DateTime<Utc>,
}
@@ -64,11 +62,11 @@ pub struct NewTransaction {
pub price: f64,
pub quality_code: i32,
pub quality_name: String,
pub card_number: String,
pub station: String,
pub terminal: String,
pub pump: String,
pub receipt: String,
pub control_number: Option<String>,
pub card_id: u32,
pub customer_id: Option<u32>,
}

20
src/db/repository.rs
View File

@@ -59,16 +59,14 @@ impl Repository {
pub async fn upsert_card(&self, card: &NewCard) -> anyhow::Result<u32> {
sqlx::query(
r#"
INSERT INTO cards (card_number, card_type, customer_id)
VALUES (?, ?, ?)
INSERT INTO cards (card_number, customer_id)
VALUES (?, ?)
ON DUPLICATE KEY UPDATE
card_type = COALESCE(VALUES(card_type), card_type),
customer_id = COALESCE(VALUES(customer_id), customer_id),
customer_id = VALUES(customer_id),
updated_at = CURRENT_TIMESTAMP
"#,
)
.bind(&card.card_number)
.bind(&card.card_type)
.bind(card.customer_id)
.execute(&self.pool)
.await?;
@@ -85,7 +83,7 @@ impl Repository {
pub async fn find_card_by_number(&self, card_number: &str) -> anyhow::Result<Option<Card>> {
let result = sqlx::query_as(
"SELECT id, card_number, card_type, customer_id, created_at, updated_at
"SELECT id, card_number, customer_id, created_at, updated_at
FROM cards
WHERE card_number = ?",
)
@@ -105,13 +103,13 @@ impl Repository {
}
let mut query = String::from(
"INSERT INTO transactions (transaction_date, batch_number, amount, volume, price, quality_code, quality_name, station, terminal, pump, receipt, control_number, card_id, customer_id) VALUES ",
"INSERT INTO transactions (transaction_date, batch_number, amount, volume, price, quality_code, quality_name, card_number, station, terminal, pump, receipt, control_number, customer_id) VALUES ",
);
let mut values = Vec::new();
for tx in transactions {
values.push(format!(
"('{}', '{}', {}, {}, {}, {}, '{}', '{}', '{}', '{}', '{}', {}, {}, {})",
"('{}', '{}', {}, {}, {}, {}, '{}', '{}', '{}', '{}', '{}', '{}', {}, {})",
tx.transaction_date.format("%Y-%m-%d %H:%M:%S"),
tx.batch_number,
tx.amount,
@@ -119,12 +117,12 @@ impl Repository {
tx.price,
tx.quality_code,
tx.quality_name.replace("'", "''"),
tx.card_number.replace("'", "''"),
tx.station,
tx.terminal,
tx.pump,
tx.receipt,
tx.control_number.as_ref().map(|s| format!("'{}'", s.replace("'", "''"))).unwrap_or_else(|| "NULL".to_string()),
tx.card_id,
tx.customer_id.map(|id| id.to_string()).unwrap_or_else(|| "NULL".to_string()),
));
}
@@ -145,8 +143,8 @@ impl Repository {
let result = sqlx::query_as(
r#"
SELECT t.id, t.transaction_date, t.batch_number, t.amount, t.volume, t.price,
t.quality_code, t.quality_name, t.station, t.terminal, t.pump,
t.receipt, t.control_number, t.card_id, t.customer_id, t.created_at
t.quality_code, t.quality_name, t.card_number, t.station, t.terminal,
t.pump, t.receipt, t.control_number, t.customer_id, t.created_at
FROM transactions t
JOIN customers c ON t.customer_id = c.id
WHERE c.customer_number = ?

10
src/main.rs
View File

@@ -391,22 +391,27 @@ async fn main() -> anyhow::Result<()> {
eprintln!("Usage: {} db <subcommand> [--env <name>]", clean_args[0]);
eprintln!("Subcommands:");
eprintln!(" setup Create database and schema");
eprintln!(" reset Drop and recreate database");
std::process::exit(1);
}
println!("Environment: {}", env.as_str());
let config = Config::load(env)?;
let pool = create_pool(&config.database.connection_url()).await?;
let repo = Repository::new(pool);
match clean_args[2].as_str() {
"setup" => {
let pool = create_pool(&config.database.connection_url()).await?;
let repo = Repository::new(pool);
commands::run_db_setup(&repo, &config).await?;
}
"reset" => {
commands::run_db_reset(&config).await?;
}
_ => {
eprintln!("Unknown db subcommand: {}", clean_args[2]);
eprintln!("Subcommands:");
eprintln!(" setup Create database and schema");
eprintln!(" reset Drop and recreate database");
std::process::exit(1);
}
}
@@ -431,6 +436,7 @@ fn print_usage(program: &str) {
eprintln!(" import <csv-file> [--env <name>] Import CSV data to database (default: prod)");
eprintln!(" generate <csv> <dir> Generate HTML invoices from CSV");
eprintln!(" db setup [--env <name>] Create database and schema (default: prod)");
eprintln!(" db reset [--env <name>] Drop and recreate database (default: prod)");
eprintln!(" help Show this help message");
eprintln!();
eprintln!("Environments: prod (default), dev, test");