From 3780fb31369c67cb0a2fffde7c2edfb395662577 Mon Sep 17 00:00:00 2001 From: n08i40k Date: Thu, 25 Sep 2025 03:14:39 +0400 Subject: [PATCH] feat(downloader): implement etag-based difference check for schedule --- .../src/updater.rs | 16 +++++----- .../src/xls_downloader.rs | 32 ++++++++++++++----- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/providers/provider-engels-polytechnic/src/updater.rs b/providers/provider-engels-polytechnic/src/updater.rs index bc50aa7..4af07a2 100644 --- a/providers/provider-engels-polytechnic/src/updater.rs +++ b/providers/provider-engels-polytechnic/src/updater.rs @@ -51,9 +51,9 @@ pub mod error { /// Errors that may occur during the creation of a schedule snapshot. #[derive(Debug, Display, Error)] pub enum SnapshotCreationError { - /// The URL is the same as the one already being used (no update needed). - #[display("The URL is the same as the one already being used.")] - SameUrl, + /// The ETag is the same (no update needed). + #[display("The ETag is the same.")] + Same, /// The URL query for the XLS file failed to execute, either due to network issues or invalid API parameters. #[display("Failed to fetch URL: {_0}")] @@ -86,10 +86,6 @@ impl Updater { downloader: &mut XlsDownloader, url: String, ) -> Result { - if downloader.url.as_ref().is_some_and(|_url| _url.eq(&url)) { - return Err(SnapshotCreationError::SameUrl); - } - let head_result = downloader.set_url(&url).await.map_err(|error| { if let FetchError::Unknown(error) = &error { sentry::capture_error(&error); @@ -98,6 +94,10 @@ impl Updater { SnapshotCreationError::FetchFailed(error) })?; + if downloader.etag == Some(head_result.etag) { + return Err(SnapshotCreationError::Same); + } + let xls_data = downloader .fetch(false) .await @@ -249,7 +249,7 @@ impl Updater { let snapshot = match Self::new_snapshot(&mut self.downloader, url).await { Ok(snapshot) => snapshot, - Err(SnapshotCreationError::SameUrl) => { + Err(SnapshotCreationError::Same) => { let mut clone = current_snapshot.clone(); clone.update(); diff --git a/providers/provider-engels-polytechnic/src/xls_downloader.rs b/providers/provider-engels-polytechnic/src/xls_downloader.rs index c188575..eb9607e 100644 --- a/providers/provider-engels-polytechnic/src/xls_downloader.rs +++ b/providers/provider-engels-polytechnic/src/xls_downloader.rs @@ -66,25 +66,30 @@ pub struct FetchOk { /// Date data received. pub requested_at: DateTime, + /// Etag. + pub etag: String, + /// File data. pub data: Option>, } impl FetchOk { /// Result without file content. - pub fn head(uploaded_at: DateTime) -> Self { + pub fn head(uploaded_at: DateTime, etag: String) -> Self { FetchOk { uploaded_at, requested_at: Utc::now(), + etag, data: None, } } /// Full result. - pub fn get(uploaded_at: DateTime, data: Vec) -> Self { + pub fn get(uploaded_at: DateTime, etag: String, data: Vec) -> Self { FetchOk { uploaded_at, requested_at: Utc::now(), + etag, data: Some(data), } } @@ -94,11 +99,15 @@ pub type FetchResult = Result; pub struct XlsDownloader { pub url: Option, + pub etag: Option, } impl XlsDownloader { pub fn new() -> Self { - XlsDownloader { url: None } + XlsDownloader { + url: None, + etag: None, + } } async fn fetch_specified(url: &str, head: bool) -> FetchResult { @@ -124,9 +133,12 @@ impl XlsDownloader { .get("Content-Type") .ok_or(FetchError::bad_headers("Content-Type"))?; - if !headers.contains_key("etag") { - return Err(FetchError::bad_headers("etag")); - } + let etag = headers + .get("etag") + .ok_or(FetchError::bad_headers("etag"))? + .to_str() + .or(Err(FetchError::bad_headers("etag")))? + .to_string(); let last_modified = headers .get("last-modified") @@ -141,9 +153,13 @@ impl XlsDownloader { .with_timezone(&Utc); Ok(if head { - FetchOk::head(last_modified) + FetchOk::head(last_modified, etag) } else { - FetchOk::get(last_modified, response.bytes().await.unwrap().to_vec()) + FetchOk::get( + last_modified, + etag, + response.bytes().await.unwrap().to_vec(), + ) }) }