From bad4ae4827022eed4486e2e0ff20b040e720b23e Mon Sep 17 00:00:00 2001 From: n08i40k Date: Tue, 29 Sep 2026 06:53:14 +0400 Subject: [PATCH] feat(downloader): download schedule from dropbox --- .../provider-engels-polytechnic/Cargo.toml | 2 +- .../src/updater/mod.rs | 12 +- .../src/xls_downloader/dropbox.rs | 202 ++++++++++++++++++ .../src/xls_downloader/mod.rs | 42 +++- .../src/xls_downloader/yandex_disk.rs | 19 +- src/state/env/schedule.rs | 36 +++- src/state/mod.rs | 13 +- 7 files changed, 292 insertions(+), 34 deletions(-) create mode 100644 providers/provider-engels-polytechnic/src/xls_downloader/dropbox.rs diff --git a/providers/provider-engels-polytechnic/Cargo.toml b/providers/provider-engels-polytechnic/Cargo.toml index fd8f22a..de80be5 100644 --- a/providers/provider-engels-polytechnic/Cargo.toml +++ b/providers/provider-engels-polytechnic/Cargo.toml @@ -21,7 +21,7 @@ utoipa = { version = "5", features = ["macros", "chrono"] } calamine = "0" async-trait = "0" -reqwest = { version = "0", features = ["json"] } +reqwest = { version = "0", features = ["json", "form"] } serde = { version = "1", features = ["derive"] } percent-encoding = "2" ua_generator = "0" diff --git a/providers/provider-engels-polytechnic/src/updater/mod.rs b/providers/provider-engels-polytechnic/src/updater/mod.rs index b8c2312..7d9f7dd 100644 --- a/providers/provider-engels-polytechnic/src/updater/mod.rs +++ b/providers/provider-engels-polytechnic/src/updater/mod.rs @@ -12,6 +12,11 @@ pub enum UpdateSource { YandexDisk { public_url: String, }, + + /// Public Dropbox folder the college uploads the schedule to. + Dropbox { + public_url: String, + }, } pub struct Updater { @@ -26,7 +31,12 @@ impl Updater { fn source(&self) -> Option { match &self.update_source { UpdateSource::Prepared(_) => None, - UpdateSource::YandexDisk { public_url } => Some(Source::new(public_url.clone())), + UpdateSource::YandexDisk { public_url } => Some(Source::YandexDisk { + public_url: public_url.clone(), + }), + UpdateSource::Dropbox { public_url } => Some(Source::Dropbox { + public_url: public_url.clone(), + }), } } diff --git a/providers/provider-engels-polytechnic/src/xls_downloader/dropbox.rs b/providers/provider-engels-polytechnic/src/xls_downloader/dropbox.rs new file mode 100644 index 0000000..b745959 --- /dev/null +++ b/providers/provider-engels-polytechnic/src/xls_downloader/dropbox.rs @@ -0,0 +1,202 @@ +use super::{FetchError, FetchResult, RemoteFile}; +use chrono::{DateTime, Utc}; +use reqwest::Url; +use serde::Deserialize; +use std::sync::Arc; + +/// Endpoint the Dropbox web interface lists shared folder entries with. +const LISTING_URL: &str = "https://www.dropbox.com/list_shared_link_folder_entries"; + +/// CSRF token sent both as a cookie and as a form field. +/// +/// Dropbox only checks that the two values match, so any token is accepted. +const CSRF_TOKEN: &str = "schedule-parser-rusted"; + +/// Parts of a shared folder link (`https://www.dropbox.com/scl/fo//?rlkey=`) +/// required for the folder listing. +#[derive(Debug, PartialEq)] +struct FolderLink { + link_key: String, + secure_hash: String, + rlkey: String, +} + +impl FolderLink { + fn parse(public_url: &str) -> FetchResult { + let url = Url::parse(public_url).map_err(|_| FetchError::InvalidUrl)?; + + if !url + .host_str() + .is_some_and(|host| host == "dropbox.com" || host.ends_with(".dropbox.com")) + { + return Err(FetchError::InvalidUrl); + } + + let segments: Vec<&str> = url + .path_segments() + .map(|segments| segments.filter(|segment| !segment.is_empty()).collect()) + .unwrap_or_default(); + + let ["scl", "fo", link_key, secure_hash] = segments.as_slice() else { + return Err(FetchError::InvalidUrl); + }; + + let rlkey = url + .query_pairs() + .find(|(key, _)| key == "rlkey") + .map(|(_, value)| value.into_owned()) + .ok_or(FetchError::InvalidUrl)?; + + Ok(Self { + link_key: link_key.to_string(), + secure_hash: secure_hash.to_string(), + rlkey, + }) + } +} + +#[derive(Deserialize)] +struct Listing { + entries: Vec, +} + +#[derive(Deserialize)] +struct Entry { + filename: String, + is_dir: bool, + href: Option, + revision_id: Option, + ts: Option, +} + +impl Entry { + /// Whether the entry is the schedule the provider is interested in. + fn is_schedule(&self) -> bool { + !self.is_dir && self.href.is_some() && super::is_schedule_name(&self.filename) + } + + fn modified_at(&self) -> DateTime { + self.ts + .and_then(|ts| DateTime::from_timestamp(ts, 0)) + .unwrap_or_default() + } +} + +/// Link to the direct download of a file, built from its preview link. +fn direct_download_url(href: &str) -> FetchResult { + let mut url = Url::parse(href).map_err(|_| FetchError::InvalidUrl)?; + + let pairs: Vec<(String, String)> = url + .query_pairs() + .filter(|(key, _)| key != "dl") + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect(); + + url.query_pairs_mut() + .clear() + .extend_pairs(pairs) + .append_pair("dl", "1"); + + Ok(url.to_string()) +} + +/// Finds the freshest schedule file in the public folder. +/// +/// The files inside the folder are replaced independently of the folder link, +/// so the whole listing is re-read on every probe. +pub async fn probe(public_url: &str) -> FetchResult { + let link = FolderLink::parse(public_url)?; + + let response = reqwest::Client::new() + .post(LISTING_URL) + .header("User-Agent", ua_generator::ua::spoof_chrome_ua()) + .header("Cookie", format!("t={CSRF_TOKEN}")) + .form(&[ + ("is_xhr", "true"), + ("t", CSRF_TOKEN), + ("link_key", &link.link_key), + ("link_type", "c"), + ("secure_hash", &link.secure_hash), + ("sub_path", ""), + ("rlkey", &link.rlkey), + ]) + .send() + .await + .map_err(|error| FetchError::unknown(Arc::new(error)))?; + + if response.status().as_u16() != 200 { + return Err(FetchError::bad_status_code(response.status().as_u16())); + } + + let listing = response + .json::() + .await + .map_err(|error| FetchError::unknown(Arc::new(error)))?; + + let entry = listing + .entries + .into_iter() + .filter(Entry::is_schedule) + .max_by_key(|entry| entry.ts) + .ok_or(FetchError::NoScheduleFile)?; + + let modified_at = entry.modified_at(); + let href = entry.href.unwrap(); + + Ok(RemoteFile { + download_url: direct_download_url(&href)?, + version: entry + .revision_id + .unwrap_or_else(|| modified_at.to_rfc3339()), + modified_at, + url: href, + }) +} + +#[cfg(test)] +mod tests { + use super::{FolderLink, direct_download_url, probe}; + + const PUBLIC_URL: &str = "https://www.dropbox.com/scl/fo/rkcg08aekpdat5sjg0er8/AOESrLwGCK3apE4oGqS4F_k?rlkey=jrv7d1kvq3lc6b7jxuj3uek41&st=uz87ct9v&dl=0"; + + #[test] + fn parse_link() { + assert_eq!( + FolderLink::parse(PUBLIC_URL).unwrap(), + FolderLink { + link_key: "rkcg08aekpdat5sjg0er8".to_string(), + secure_hash: "AOESrLwGCK3apE4oGqS4F_k".to_string(), + rlkey: "jrv7d1kvq3lc6b7jxuj3uek41".to_string(), + } + ); + + assert!(FolderLink::parse("https://www.dropbox.com/scl/fo/key/hash").is_err()); + assert!(FolderLink::parse("https://disk.yandex.ru/d/e8HJpMgDq7msyg").is_err()); + } + + #[test] + fn download_url() { + assert_eq!( + direct_download_url("https://www.dropbox.com/scl/fo/a/b/f.xls?rlkey=c&dl=0").unwrap(), + "https://www.dropbox.com/scl/fo/a/b/f.xls?rlkey=c&dl=1" + ); + } + + #[tokio::test] + async fn probe_ok() { + let file = probe(PUBLIC_URL).await.unwrap(); + + assert!(file.url.starts_with("https://www.dropbox.com/scl/fo/")); + assert!(!file.version.is_empty()); + assert!(file.download_url.ends_with("dl=1")); + } + + #[tokio::test] + async fn probe_unknown_folder() { + assert!( + probe("https://www.dropbox.com/scl/fo/000000000000000000000/AAAAAAAAAAAAAAAAAAAAAAA?rlkey=0000000000000000000000000") + .await + .is_err() + ); + } +} diff --git a/providers/provider-engels-polytechnic/src/xls_downloader/mod.rs b/providers/provider-engels-polytechnic/src/xls_downloader/mod.rs index 19e6bdc..bf577f6 100644 --- a/providers/provider-engels-polytechnic/src/xls_downloader/mod.rs +++ b/providers/provider-engels-polytechnic/src/xls_downloader/mod.rs @@ -3,8 +3,27 @@ use derive_more::{Display, Error}; use std::mem::discriminant; use std::sync::Arc; +mod dropbox; mod yandex_disk; +/// Prefix of the schedule file names in the shared folder. +const NAME_PREFIX: &str = "poltavskaja_"; + +/// Marker of the corrections file, which holds a separate schedule. +const NAME_EXCLUDED_MARKER: &str = "korr"; + +/// Extension of the schedule files. +const NAME_SUFFIX: &str = ".xls"; + +/// Whether the file name belongs to the schedule the provider is interested in. +fn is_schedule_name(name: &str) -> bool { + let name = name.to_lowercase(); + + name.starts_with(NAME_PREFIX) + && name.ends_with(NAME_SUFFIX) + && !name.contains(NAME_EXCLUDED_MARKER) +} + /// XLS data retrieval errors. #[derive(Clone, Debug, Display, Error)] pub enum FetchError { @@ -16,6 +35,10 @@ pub enum FetchError { #[display("Server returned a status code {status_code}.")] BadStatusCode { status_code: u16 }, + /// The shared folder link has an unsupported format. + #[display("The shared folder link has an unsupported format.")] + InvalidUrl, + /// The folder contains no file matching the schedule name pattern. #[display("No schedule file was found in the shared folder.")] NoScheduleFile, @@ -55,20 +78,23 @@ pub struct RemoteFile { pub modified_at: DateTime, } -/// Public Yandex Disk folder the schedule is downloaded from. +/// Public shared folder the schedule is downloaded from. #[derive(Clone, Debug)] -pub struct Source { - pub public_url: String, +pub enum Source { + /// Public Yandex Disk folder. + YandexDisk { public_url: String }, + + /// Public Dropbox folder. + Dropbox { public_url: String }, } impl Source { - pub fn new(public_url: String) -> Self { - Self { public_url } - } - /// Looks up the current schedule file without downloading its content. pub async fn probe(&self) -> FetchResult { - yandex_disk::probe(&self.public_url).await + match self { + Self::YandexDisk { public_url } => yandex_disk::probe(public_url).await, + Self::Dropbox { public_url } => dropbox::probe(public_url).await, + } } /// Downloads the content of a previously probed file. diff --git a/providers/provider-engels-polytechnic/src/xls_downloader/yandex_disk.rs b/providers/provider-engels-polytechnic/src/xls_downloader/yandex_disk.rs index 553eec5..a209f87 100644 --- a/providers/provider-engels-polytechnic/src/xls_downloader/yandex_disk.rs +++ b/providers/provider-engels-polytechnic/src/xls_downloader/yandex_disk.rs @@ -3,15 +3,6 @@ use chrono::{DateTime, Utc}; use percent_encoding::{AsciiSet, CONTROLS, NON_ALPHANUMERIC, utf8_percent_encode}; use serde::Deserialize; -/// Prefix of the schedule file names in the shared folder. -const NAME_PREFIX: &str = "poltavskaja_"; - -/// Marker of the corrections file, which holds a separate schedule. -const NAME_EXCLUDED_MARKER: &str = "korr"; - -/// Extension of the schedule files. -const NAME_SUFFIX: &str = ".xls"; - /// Maximum amount of entries requested from the folder listing. const LISTING_LIMIT: u32 = 200; @@ -54,15 +45,7 @@ struct Item { impl Item { /// Whether the entry is the schedule the provider is interested in. fn is_schedule(&self) -> bool { - if self.resource_type != "file" || self.file.is_none() { - return false; - } - - let name = self.name.to_lowercase(); - - name.starts_with(NAME_PREFIX) - && name.ends_with(NAME_SUFFIX) - && !name.contains(NAME_EXCLUDED_MARKER) + self.resource_type == "file" && self.file.is_some() && super::is_schedule_name(&self.name) } /// Marker changing whenever the file content changes. diff --git a/src/state/env/schedule.rs b/src/state/env/schedule.rs index 8656821..0687ca3 100644 --- a/src/state/env/schedule.rs +++ b/src/state/env/schedule.rs @@ -1,10 +1,39 @@ use std::env; +/// Public shared folder the schedule files are uploaded to. +#[cfg(not(test))] +#[derive(Clone)] +pub enum ScheduleSourceUrl { + /// Public link to a Yandex Disk folder (`SCHEDULE_YANDEX_DISK_URL`). + YandexDisk(String), + + /// Public link to a Dropbox folder (`SCHEDULE_DROPBOX_URL`). + Dropbox(String), +} + +#[cfg(not(test))] +impl ScheduleSourceUrl { + fn from_env() -> Self { + match ( + env::var("SCHEDULE_YANDEX_DISK_URL").ok(), + env::var("SCHEDULE_DROPBOX_URL").ok(), + ) { + (Some(url), None) => Self::YandexDisk(url), + (None, Some(url)) => Self::Dropbox(url), + (Some(_), Some(_)) => { + panic!("Only one of SCHEDULE_YANDEX_DISK_URL and SCHEDULE_DROPBOX_URL must be set") + } + (None, None) => { + panic!("SCHEDULE_YANDEX_DISK_URL or SCHEDULE_DROPBOX_URL must be set") + } + } + } +} + #[derive(Clone)] pub struct ScheduleEnvData { - /// Public link to the Yandex Disk folder the schedule files are uploaded to. #[cfg(not(test))] - pub yandex_disk_url: String, + pub source_url: ScheduleSourceUrl, pub auto_update: bool, } @@ -13,8 +42,7 @@ impl Default for ScheduleEnvData { fn default() -> Self { Self { #[cfg(not(test))] - yandex_disk_url: env::var("SCHEDULE_YANDEX_DISK_URL") - .expect("SCHEDULE_YANDEX_DISK_URL must be set"), + source_url: ScheduleSourceUrl::from_env(), auto_update: !env::var("SCHEDULE_DISABLE_AUTO_UPDATE") .is_ok_and(|v| v.eq("1") || v.eq("true")), } diff --git a/src/state/mod.rs b/src/state/mod.rs index a3d4c28..b27ef94 100644 --- a/src/state/mod.rs +++ b/src/state/mod.rs @@ -38,8 +38,17 @@ impl AppState { #[cfg(not(test))] { - providers::EngelsPolytechnicUpdateSource::YandexDisk { - public_url: env.schedule.yandex_disk_url.clone(), + match &env.schedule.source_url { + env::schedule::ScheduleSourceUrl::YandexDisk(public_url) => { + providers::EngelsPolytechnicUpdateSource::YandexDisk { + public_url: public_url.clone(), + } + } + env::schedule::ScheduleSourceUrl::Dropbox(public_url) => { + providers::EngelsPolytechnicUpdateSource::Dropbox { + public_url: public_url.clone(), + } + } } } })