From 9c4514208304df02de2c536917558d1b10e6daa1 Mon Sep 17 00:00:00 2001 From: dolphinau Date: Mon, 28 Jul 2025 18:47:16 +0200 Subject: [PATCH] Add db support to store articles --- .envrc | 0 .gitignore | 0 Cargo.toml | 6 ++-- flake.nix | 6 +--- justfile | 7 +++++ src/main.rs | 79 +++++++++++++++++++++++++++++++++++++++++++---------- 6 files changed, 75 insertions(+), 23 deletions(-) mode change 100644 => 100755 .envrc mode change 100644 => 100755 .gitignore mode change 100644 => 100755 Cargo.toml mode change 100644 => 100755 flake.nix create mode 100644 justfile mode change 100644 => 100755 src/main.rs diff --git a/.envrc b/.envrc old mode 100644 new mode 100755 diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/Cargo.toml b/Cargo.toml old mode 100644 new mode 100755 index 19d52b5..2b9e573 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,12 +4,12 @@ version = "0.1.0" edition = "2024" [dependencies] -tokio = { version = "1.46.0", default-features = false, features = [ - "rt-multi-thread", -] } +tokio = { version = "1.46.0", features = ["full"] } futures = "0.3.31" reqwest = "0.12.22" scraper = "0.23.1" regex = "1.11.1" chrono = "0.4.41" rss = "2.0.12" +mini-redis = "0.4.1" +tokio-postgres = "0.7.13" diff --git a/flake.nix b/flake.nix old mode 100644 new mode 100755 index 5b3a28b..8e666d5 --- a/flake.nix +++ b/flake.nix @@ -1,6 +1,5 @@ { description = "lwn-sub-snoozer"; - inputs = { flake-utils.url = "github:numtide/flake-utils"; nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05"; @@ -25,15 +24,12 @@ # $ nix develop devShells.default = pkgs.mkShell { LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [pkgs.openssl]; - packages = [ - pkgs.gnumake pkgs.pkg-config pkgs.openssl - - # Nix pkgs.nixpkgs-fmt pkgs.nil + pkgs.postgresql # Rust fenix.packages.${system}.default.toolchain diff --git a/justfile b/justfile new file mode 100644 index 0000000..00cdc3d --- /dev/null +++ b/justfile @@ -0,0 +1,7 @@ +clean: + rm -rf target + podman network rm lwn-sub-snoozer || true + +db: + podman network create lwn-sub-snoozer || true + podman run --name postgres --rm -p 5432:5432 --network=lwn-sub-snoozer -e POSTGRES_DB=dev -e POSTGRES_USER=root -e POSTGRES_PASSWORD=root docker.io/postgres:alpine diff --git a/src/main.rs b/src/main.rs old mode 100644 new mode 100755 index 62cc96f..e9aecaf --- a/src/main.rs +++ b/src/main.rs @@ -3,21 +3,72 @@ use std::error::Error; use chrono::NaiveDate; use regex::Regex; use reqwest::get; -use rss::Channel; use scraper::{Html, Selector}; -use tokio::{runtime::Runtime, sync::mpsc::unbounded_channel}; +use tokio::{ + runtime::Runtime, + sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}, +}; +use tokio_postgres; fn main() { let rt = Runtime::new().unwrap(); rt.block_on(async { - if let Ok(articles) = fetch_paid_article_urls().await { - for article in articles { - if let Ok(Some(date)) = fetch_release_date(&article).await { - // TODO - println!("Snooze {} to {}", article, date); + // Connect to the database. + if let Ok((client, connection)) = tokio_postgres::connect( + "host=localhost dbname=dev user=root password=root", + tokio_postgres::NoTls, + ) + .await + { + tokio::spawn(async move { + if let Err(e) = connection.await { + eprintln!("connection error: {}", e); + } + }); + + // Get new [$] articles + if let Ok(items) = fetch_paid_article_urls().await { + for item in items { + if let Some(link) = item.link() { + match client + .query_opt("SELECT date FROM articles WHERE id = $1", &[&link]) + .await + { + Ok(None) => { + if let Ok(Some(date)) = fetch_release_date(&link).await { + println!("Adding new article to db: {}", link); + if let Err(e) = client + .query( + "INSERT INTO articles (id, date) VALUES ($1, $2)", + &[&link, &date.to_string()], + ) + .await + { + eprintln!("Error insert: {}", e); + } + } + } + _ => (), + } + }; } } + + // TODO: Check for new free articles + // client + // .query("SELECT * FROM articles") + // .await + // .unwrap() + // .iter() + // .map(|row| { + // let id = row.get("id"); + // let date = row.get("date"); + // + // if date < today { + // article.publish + // } + // }) } }); } @@ -31,12 +82,11 @@ async fn fetch_release_date(url: &str) -> Result, Box Result, Box Result, Box> { +async fn fetch_paid_article_urls() -> Result, Box> { let response = get("https://lwn.net/headlines/rss").await?.bytes().await?; - let channel = Channel::read_from(&response[..])?; + let channel = rss::Channel::read_from(&response[..])?; Ok(channel .items() .iter() .filter(|i| i.title().unwrap_or("").starts_with("[$]")) - .filter_map(|i| i.link()) - .map(|s| s.to_string()) - .collect::>()) + .map(|i| i.clone()) + .collect::>()) }