feat(downloader): download schedule from dropbox

providers/provider-engels-polytechnic/Cargo.toml

@@ -21,7 +21,7 @@ utoipa = { version = "5", features = ["macros", "chrono"] }
 calamine = "0"
 async-trait = "0"
 
-reqwest = { version = "0", features = ["json"] }
+reqwest = { version = "0", features = ["json", "form"] }
 serde = { version = "1", features = ["derive"] }
 percent-encoding = "2"
 ua_generator = "0"

providers/provider-engels-polytechnic/src/updater/mod.rs

@@ -12,6 +12,11 @@ pub enum UpdateSource {
     YandexDisk {
         public_url: String,
     },
+
+    /// Public Dropbox folder the college uploads the schedule to.
+    Dropbox {
+        public_url: String,
+    },
 }
 
 pub struct Updater {
@@ -26,7 +31,12 @@ impl Updater {
     fn source(&self) -> Option<Source> {
         match &self.update_source {
             UpdateSource::Prepared(_) => None,
-            UpdateSource::YandexDisk { public_url } => Some(Source::new(public_url.clone())),
+            UpdateSource::YandexDisk { public_url } => Some(Source::YandexDisk {
+                public_url: public_url.clone(),
+            }),
+            UpdateSource::Dropbox { public_url } => Some(Source::Dropbox {
+                public_url: public_url.clone(),
+            }),
         }
     }
 

providers/provider-engels-polytechnic/src/xls_downloader/dropbox.rs

@@ -0,0 +1,202 @@
+use super::{FetchError, FetchResult, RemoteFile};
+use chrono::{DateTime, Utc};
+use reqwest::Url;
+use serde::Deserialize;
+use std::sync::Arc;
+
+/// Endpoint the Dropbox web interface lists shared folder entries with.
+const LISTING_URL: &str = "https://www.dropbox.com/list_shared_link_folder_entries";
+
+/// CSRF token sent both as a cookie and as a form field.
+///
+/// Dropbox only checks that the two values match, so any token is accepted.
+const CSRF_TOKEN: &str = "schedule-parser-rusted";
+
+/// Parts of a shared folder link (`https://www.dropbox.com/scl/fo/<key>/<hash>?rlkey=<rlkey>`)
+/// required for the folder listing.
+#[derive(Debug, PartialEq)]
+struct FolderLink {
+    link_key: String,
+    secure_hash: String,
+    rlkey: String,
+}
+
+impl FolderLink {
+    fn parse(public_url: &str) -> FetchResult<Self> {
+        let url = Url::parse(public_url).map_err(|_| FetchError::InvalidUrl)?;
+
+        if !url
+            .host_str()
+            .is_some_and(|host| host == "dropbox.com" || host.ends_with(".dropbox.com"))
+        {
+            return Err(FetchError::InvalidUrl);
+        }
+
+        let segments: Vec<&str> = url
+            .path_segments()
+            .map(|segments| segments.filter(|segment| !segment.is_empty()).collect())
+            .unwrap_or_default();
+
+        let ["scl", "fo", link_key, secure_hash] = segments.as_slice() else {
+            return Err(FetchError::InvalidUrl);
+        };
+
+        let rlkey = url
+            .query_pairs()
+            .find(|(key, _)| key == "rlkey")
+            .map(|(_, value)| value.into_owned())
+            .ok_or(FetchError::InvalidUrl)?;
+
+        Ok(Self {
+            link_key: link_key.to_string(),
+            secure_hash: secure_hash.to_string(),
+            rlkey,
+        })
+    }
+}
+
+#[derive(Deserialize)]
+struct Listing {
+    entries: Vec<Entry>,
+}
+
+#[derive(Deserialize)]
+struct Entry {
+    filename: String,
+    is_dir: bool,
+    href: Option<String>,
+    revision_id: Option<String>,
+    ts: Option<i64>,
+}
+
+impl Entry {
+    /// Whether the entry is the schedule the provider is interested in.
+    fn is_schedule(&self) -> bool {
+        !self.is_dir && self.href.is_some() && super::is_schedule_name(&self.filename)
+    }
+
+    fn modified_at(&self) -> DateTime<Utc> {
+        self.ts
+            .and_then(|ts| DateTime::from_timestamp(ts, 0))
+            .unwrap_or_default()
+    }
+}
+
+/// Link to the direct download of a file, built from its preview link.
+fn direct_download_url(href: &str) -> FetchResult<String> {
+    let mut url = Url::parse(href).map_err(|_| FetchError::InvalidUrl)?;
+
+    let pairs: Vec<(String, String)> = url
+        .query_pairs()
+        .filter(|(key, _)| key != "dl")
+        .map(|(key, value)| (key.into_owned(), value.into_owned()))
+        .collect();
+
+    url.query_pairs_mut()
+        .clear()
+        .extend_pairs(pairs)
+        .append_pair("dl", "1");
+
+    Ok(url.to_string())
+}
+
+/// Finds the freshest schedule file in the public folder.
+///
+/// The files inside the folder are replaced independently of the folder link,
+/// so the whole listing is re-read on every probe.
+pub async fn probe(public_url: &str) -> FetchResult<RemoteFile> {
+    let link = FolderLink::parse(public_url)?;
+
+    let response = reqwest::Client::new()
+        .post(LISTING_URL)
+        .header("User-Agent", ua_generator::ua::spoof_chrome_ua())
+        .header("Cookie", format!("t={CSRF_TOKEN}"))
+        .form(&[
+            ("is_xhr", "true"),
+            ("t", CSRF_TOKEN),
+            ("link_key", &link.link_key),
+            ("link_type", "c"),
+            ("secure_hash", &link.secure_hash),
+            ("sub_path", ""),
+            ("rlkey", &link.rlkey),
+        ])
+        .send()
+        .await
+        .map_err(|error| FetchError::unknown(Arc::new(error)))?;
+
+    if response.status().as_u16() != 200 {
+        return Err(FetchError::bad_status_code(response.status().as_u16()));
+    }
+
+    let listing = response
+        .json::<Listing>()
+        .await
+        .map_err(|error| FetchError::unknown(Arc::new(error)))?;
+
+    let entry = listing
+        .entries
+        .into_iter()
+        .filter(Entry::is_schedule)
+        .max_by_key(|entry| entry.ts)
+        .ok_or(FetchError::NoScheduleFile)?;
+
+    let modified_at = entry.modified_at();
+    let href = entry.href.unwrap();
+
+    Ok(RemoteFile {
+        download_url: direct_download_url(&href)?,
+        version: entry
+            .revision_id
+            .unwrap_or_else(|| modified_at.to_rfc3339()),
+        modified_at,
+        url: href,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{FolderLink, direct_download_url, probe};
+
+    const PUBLIC_URL: &str = "https://www.dropbox.com/scl/fo/rkcg08aekpdat5sjg0er8/AOESrLwGCK3apE4oGqS4F_k?rlkey=jrv7d1kvq3lc6b7jxuj3uek41&st=uz87ct9v&dl=0";
+
+    #[test]
+    fn parse_link() {
+        assert_eq!(
+            FolderLink::parse(PUBLIC_URL).unwrap(),
+            FolderLink {
+                link_key: "rkcg08aekpdat5sjg0er8".to_string(),
+                secure_hash: "AOESrLwGCK3apE4oGqS4F_k".to_string(),
+                rlkey: "jrv7d1kvq3lc6b7jxuj3uek41".to_string(),
+            }
+        );
+
+        assert!(FolderLink::parse("https://www.dropbox.com/scl/fo/key/hash").is_err());
+        assert!(FolderLink::parse("https://disk.yandex.ru/d/e8HJpMgDq7msyg").is_err());
+    }
+
+    #[test]
+    fn download_url() {
+        assert_eq!(
+            direct_download_url("https://www.dropbox.com/scl/fo/a/b/f.xls?rlkey=c&dl=0").unwrap(),
+            "https://www.dropbox.com/scl/fo/a/b/f.xls?rlkey=c&dl=1"
+        );
+    }
+
+    #[tokio::test]
+    async fn probe_ok() {
+        let file = probe(PUBLIC_URL).await.unwrap();
+
+        assert!(file.url.starts_with("https://www.dropbox.com/scl/fo/"));
+        assert!(!file.version.is_empty());
+        assert!(file.download_url.ends_with("dl=1"));
+    }
+
+    #[tokio::test]
+    async fn probe_unknown_folder() {
+        assert!(
+            probe("https://www.dropbox.com/scl/fo/000000000000000000000/AAAAAAAAAAAAAAAAAAAAAAA?rlkey=0000000000000000000000000")
+                .await
+                .is_err()
+        );
+    }
+}

providers/provider-engels-polytechnic/src/xls_downloader/mod.rs

@@ -3,8 +3,27 @@ use derive_more::{Display, Error};
 use std::mem::discriminant;
 use std::sync::Arc;
 
+mod dropbox;
 mod yandex_disk;
 
+/// Prefix of the schedule file names in the shared folder.
+const NAME_PREFIX: &str = "poltavskaja_";
+
+/// Marker of the corrections file, which holds a separate schedule.
+const NAME_EXCLUDED_MARKER: &str = "korr";
+
+/// Extension of the schedule files.
+const NAME_SUFFIX: &str = ".xls";
+
+/// Whether the file name belongs to the schedule the provider is interested in.
+fn is_schedule_name(name: &str) -> bool {
+    let name = name.to_lowercase();
+
+    name.starts_with(NAME_PREFIX)
+        && name.ends_with(NAME_SUFFIX)
+        && !name.contains(NAME_EXCLUDED_MARKER)
+}
+
 /// XLS data retrieval errors.
 #[derive(Clone, Debug, Display, Error)]
 pub enum FetchError {
@@ -16,6 +35,10 @@ pub enum FetchError {
     #[display("Server returned a status code {status_code}.")]
     BadStatusCode { status_code: u16 },
 
+    /// The shared folder link has an unsupported format.
+    #[display("The shared folder link has an unsupported format.")]
+    InvalidUrl,
+
     /// The folder contains no file matching the schedule name pattern.
     #[display("No schedule file was found in the shared folder.")]
     NoScheduleFile,
@@ -55,20 +78,23 @@ pub struct RemoteFile {
     pub modified_at: DateTime<Utc>,
 }
 
-/// Public Yandex Disk folder the schedule is downloaded from.
+/// Public shared folder the schedule is downloaded from.
 #[derive(Clone, Debug)]
-pub struct Source {
+pub enum Source {
-    pub public_url: String,
+    /// Public Yandex Disk folder.
+    YandexDisk { public_url: String },
+
+    /// Public Dropbox folder.
+    Dropbox { public_url: String },
 }
 
 impl Source {
-    pub fn new(public_url: String) -> Self {
-        Self { public_url }
-    }
-
     /// Looks up the current schedule file without downloading its content.
     pub async fn probe(&self) -> FetchResult<RemoteFile> {
-        yandex_disk::probe(&self.public_url).await
+        match self {
+            Self::YandexDisk { public_url } => yandex_disk::probe(public_url).await,
+            Self::Dropbox { public_url } => dropbox::probe(public_url).await,
+        }
     }
 
     /// Downloads the content of a previously probed file.

providers/provider-engels-polytechnic/src/xls_downloader/yandex_disk.rs

@@ -3,15 +3,6 @@ use chrono::{DateTime, Utc};
 use percent_encoding::{AsciiSet, CONTROLS, NON_ALPHANUMERIC, utf8_percent_encode};
 use serde::Deserialize;
 
-/// Prefix of the schedule file names in the shared folder.
-const NAME_PREFIX: &str = "poltavskaja_";
-
-/// Marker of the corrections file, which holds a separate schedule.
-const NAME_EXCLUDED_MARKER: &str = "korr";
-
-/// Extension of the schedule files.
-const NAME_SUFFIX: &str = ".xls";
-
 /// Maximum amount of entries requested from the folder listing.
 const LISTING_LIMIT: u32 = 200;
 
@@ -54,15 +45,7 @@ struct Item {
 impl Item {
     /// Whether the entry is the schedule the provider is interested in.
     fn is_schedule(&self) -> bool {
-        if self.resource_type != "file" || self.file.is_none() {
+        self.resource_type == "file" && self.file.is_some() && super::is_schedule_name(&self.name)
-            return false;
-        }
-
-        let name = self.name.to_lowercase();
-
-        name.starts_with(NAME_PREFIX)
-            && name.ends_with(NAME_SUFFIX)
-            && !name.contains(NAME_EXCLUDED_MARKER)
     }
 
     /// Marker changing whenever the file content changes.

src/state/env/schedule.rs

@@ -1,10 +1,39 @@
 use std::env;
 
+/// Public shared folder the schedule files are uploaded to.
+#[cfg(not(test))]
+#[derive(Clone)]
+pub enum ScheduleSourceUrl {
+    /// Public link to a Yandex Disk folder (`SCHEDULE_YANDEX_DISK_URL`).
+    YandexDisk(String),
+
+    /// Public link to a Dropbox folder (`SCHEDULE_DROPBOX_URL`).
+    Dropbox(String),
+}
+
+#[cfg(not(test))]
+impl ScheduleSourceUrl {
+    fn from_env() -> Self {
+        match (
+            env::var("SCHEDULE_YANDEX_DISK_URL").ok(),
+            env::var("SCHEDULE_DROPBOX_URL").ok(),
+        ) {
+            (Some(url), None) => Self::YandexDisk(url),
+            (None, Some(url)) => Self::Dropbox(url),
+            (Some(_), Some(_)) => {
+                panic!("Only one of SCHEDULE_YANDEX_DISK_URL and SCHEDULE_DROPBOX_URL must be set")
+            }
+            (None, None) => {
+                panic!("SCHEDULE_YANDEX_DISK_URL or SCHEDULE_DROPBOX_URL must be set")
+            }
+        }
+    }
+}
+
 #[derive(Clone)]
 pub struct ScheduleEnvData {
-    /// Public link to the Yandex Disk folder the schedule files are uploaded to.
     #[cfg(not(test))]
-    pub yandex_disk_url: String,
+    pub source_url: ScheduleSourceUrl,
 
     pub auto_update: bool,
 }
@@ -13,8 +42,7 @@ impl Default for ScheduleEnvData {
     fn default() -> Self {
         Self {
             #[cfg(not(test))]
-            yandex_disk_url: env::var("SCHEDULE_YANDEX_DISK_URL")
+            source_url: ScheduleSourceUrl::from_env(),
-                .expect("SCHEDULE_YANDEX_DISK_URL must be set"),
             auto_update: !env::var("SCHEDULE_DISABLE_AUTO_UPDATE")
                 .is_ok_and(|v| v.eq("1") || v.eq("true")),
         }

src/state/mod.rs

@@ -38,8 +38,17 @@ impl AppState {
 
                 #[cfg(not(test))]
                 {
+                    match &env.schedule.source_url {
+                        env::schedule::ScheduleSourceUrl::YandexDisk(public_url) => {
                             providers::EngelsPolytechnicUpdateSource::YandexDisk {
-                        public_url: env.schedule.yandex_disk_url.clone(),
+                                public_url: public_url.clone(),
+                            }
+                        }
+                        env::schedule::ScheduleSourceUrl::Dropbox(public_url) => {
+                            providers::EngelsPolytechnicUpdateSource::Dropbox {
+                                public_url: public_url.clone(),
+                            }
+                        }
                     }
                 }
             })