feat(downloader): download schedule from different source

.env.test

@@ -1,5 +1,5 @@
 # Schedule
-# SCHEDULE_INIT_URL=
+SCHEDULE_YANDEX_DISK_URL="https://disk.yandex.ru/d/xxxxxxxxxxxxxx"
 SCHEDULE_DISABLE_AUTO_UPDATE=1
 
 # Basic authorization
@@ -14,13 +14,9 @@ TELEGRAM_BOT_ID=0
 TELEGRAM_MINI_APP_HOST=example.com
 TELEGRAM_TEST_DC=false
 
-# Yandex Cloud
-YANDEX_CLOUD_API_KEY=""
-YANDEX_CLOUD_FUNC_ID=""
-
 # Firebase
 # GOOGLE_APPLICATION_CREDENTIALS=
 
 # LOGGING
 RUST_BACKTRACE=1
-# RUST_LOG=debug
+# RUST_LOG=debug

.github/workflows/build.yml

@@ -51,8 +51,6 @@ jobs:
           TELEGRAM_BOT_ID: 0
           TELEGRAM_MINI_APP_HOST: example.com
           TELEGRAM_TEST_DC: false
-          YANDEX_CLOUD_API_KEY: ""
-          YANDEX_CLOUD_FUNC_ID: ""
   build:
     name: Build
     runs-on: ubuntu-latest

.github/workflows/release.yml

@@ -50,8 +50,6 @@ jobs:
           TELEGRAM_BOT_ID: 0
           TELEGRAM_MINI_APP_HOST: example.com
           TELEGRAM_TEST_DC: false
-          YANDEX_CLOUD_API_KEY: ""
-          YANDEX_CLOUD_FUNC_ID: ""
   build:
     name: Build
     runs-on: ubuntu-latest

.github/workflows/test.yml

@@ -33,5 +33,3 @@ jobs:
           TELEGRAM_BOT_ID: 0
           TELEGRAM_MINI_APP_HOST: example.com
           TELEGRAM_TEST_DC: false
-          YANDEX_CLOUD_API_KEY: ""
-          YANDEX_CLOUD_FUNC_ID: ""

providers/provider-engels-polytechnic/Cargo.toml

@@ -21,7 +21,9 @@ utoipa = { version = "5", features = ["macros", "chrono"] }
 calamine = "0"
 async-trait = "0"
 
-reqwest = "0"
+reqwest = { version = "0", features = ["json"] }
+serde = { version = "1", features = ["derive"] }
+percent-encoding = "2"
 ua_generator = "0"
 regex = "1"
 strsim = "0"

providers/provider-engels-polytechnic/src/lib.rs

@@ -63,8 +63,6 @@ impl ScheduleProvider for Wrapper {
                             this.snapshot = Arc::new(snapshot);
                         },
 
-                        Err(updater::Error::EmptyUri) => {},
-
                         Err(err) => {
                             sentry::capture_error(&err);
                         }

providers/provider-engels-polytechnic/src/updater/error.rs

@@ -3,25 +3,16 @@ use derive_more::{Display, Error, From};
 
 #[derive(Debug, Display, Error, From)]
 pub enum Error {
-    /// Occurs when the request to the Yandex Cloud API fails.
+    /// The remote file has not changed since the last update.
-    ///
+    #[display("The schedule file has not changed.")]
-    /// This may be due to network issues, invalid API key, incorrect function ID, or other
+    NotModified,
-    /// problems with the Yandex Cloud Function invocation.
-    #[display("An error occurred during the request to the Yandex Cloud API: {_0}")]
-    Reqwest(reqwest::Error),
 
-    #[display("Unable to get URI in 3 retries")]
+    /// The lookup of the current schedule file failed, either due to network issues or an
-    EmptyUri,
+    /// unexpected response from the storage.
-
-    /// The ETag is the same (no update needed).
-    #[display("The ETag is the same.")]
-    SameETag,
-
-    /// The URL query for the XLS file failed to execute, either due to network issues or invalid API parameters.
     #[display("Failed to fetch URL: {_0}")]
     ScheduleFetchFailed(FetchError),
 
-    /// Downloading the XLS file content failed after successfully obtaining the URL.
+    /// Downloading the XLS file content failed after successfully locating the file.
     #[display("Download failed: {_0}")]
     ScheduleDownloadFailed(FetchError),
 

providers/provider-engels-polytechnic/src/updater/mod.rs

@@ -1,40 +1,48 @@
 pub use self::error::{Error, Result};
 use crate::parser::parse_xls;
-use crate::xls_downloader::{FetchError, XlsDownloader};
+use crate::xls_downloader::{FetchError, Source};
 use base::ScheduleSnapshot;
+use chrono::Utc;
 mod error;
 
 pub enum UpdateSource {
     Prepared(ScheduleSnapshot),
 
-    Url(String),
+    /// Public Yandex Disk folder the college uploads the schedule to.
-
+    YandexDisk {
-    GrabFromSite {
+        public_url: String,
-        yandex_api_key: String,
-        yandex_func_id: String,
     },
 }
 
 pub struct Updater {
-    downloader: XlsDownloader,
     update_source: UpdateSource,
+
+    /// Version of the file the current snapshot was built from.
+    version: Option<String>,
 }
 
 impl Updater {
-    /// Constructs a new `ScheduleSnapshot` by downloading and parsing schedule data from the specified URL.
+    /// Place the schedule is downloaded from, or [`None`] for a prepared snapshot.
+    fn source(&self) -> Option<Source> {
+        match &self.update_source {
+            UpdateSource::Prepared(_) => None,
+            UpdateSource::YandexDisk { public_url } => Some(Source::new(public_url.clone())),
+        }
+    }
+
+    /// Constructs a new [`ScheduleSnapshot`] by downloading and parsing the current schedule file.
     ///
-    /// This method first checks if the provided URL is the same as the one already configured in the downloader.
+    /// The file is looked up first, and its content is downloaded only when the version marker
-    /// If different, it updates the downloader's URL, fetches the XLS content, parses it, and creates a snapshot.
+    /// differs from the one the current snapshot was built from.
-    /// Errors are returned for URL conflicts, network issues, download failures, or invalid data.
     ///
-    /// # Arguments
+    /// # Returns
     ///
-    /// * `downloader`: A mutable reference to an `XLSDownloader` implementation used to fetch and parse the schedule data.
+    /// Returns [`Error::NotModified`] when the remote file has not changed since the last update,
-    /// * `url`: The source URL pointing to the XLS file containing schedule data.
+    /// or an error describing the failed download or parsing.
-    ///
+    async fn new_snapshot(&mut self) -> Result<ScheduleSnapshot> {
-    /// returns: Result<ScheduleSnapshot, SnapshotCreationError>
+        let source = self.source().expect("a prepared snapshot has no source");
-    async fn new_snapshot(downloader: &mut XlsDownloader, url: String) -> Result<ScheduleSnapshot> {
+
-        let head_result = downloader.set_url(&url).await.map_err(|error| {
+        let file = source.probe().await.map_err(|error| {
             if let FetchError::Reqwest(error) = &error {
                 sentry::capture_error(&error);
             }
@@ -42,110 +50,44 @@ impl Updater {
             Error::ScheduleFetchFailed(error)
         })?;
 
-        if downloader.etag == Some(head_result.etag) {
+        if self.version.as_deref() == Some(file.version.as_str()) {
-            return Err(Error::SameETag);
+            return Err(Error::NotModified);
         }
 
-        let xls_data = downloader
+        let xls_data = source.download(&file).await.map_err(|error| {
-            .fetch(false)
+            if let FetchError::Reqwest(error) = &error {
-            .await
+                sentry::capture_error(&error);
-            .map_err(|error| {
+            }
-                if let FetchError::Reqwest(error) = &error {
-                    sentry::capture_error(&error);
-                }
 
-                Error::ScheduleDownloadFailed(error)
+            Error::ScheduleDownloadFailed(error)
-            })?
+        })?;
-            .data
-            .unwrap();
 
         let parse_result = parse_xls(&xls_data)?;
 
+        self.version = Some(file.version);
+
         Ok(ScheduleSnapshot {
-            fetched_at: head_result.requested_at,
+            fetched_at: Utc::now(),
-            updated_at: head_result.uploaded_at,
+            updated_at: file.modified_at,
-            url,
+            url: file.url,
             data: parse_result,
         })
     }
 
-    /// Queries the Yandex Cloud Function (FaaS) to obtain a URL for the schedule file.
+    /// Initializes the schedule by downloading the current file from the configured source.
-    ///
-    /// This sends a POST request to the specified Yandex Cloud Function endpoint,
-    /// using the provided API key for authentication. The returned URI is combined
-    /// with the "https://politehnikum-eng.ru" base domain to form the complete URL.
     ///
     /// # Arguments
     ///
-    /// * `api_key` - Authentication token for Yandex Cloud API
+    /// * `update_source`: Place the schedule is taken from.
-    /// * `func_id` - ID of the target Yandex Cloud Function to invoke
     ///
     /// # Returns
     ///
-    /// Result containing:
+    /// Returns the updater together with the initial [`ScheduleSnapshot`], or an error if the
-    /// - `Ok(String)` - Complete URL constructed from the Function's response
+    /// schedule could not be downloaded or parsed.
-    /// - `Err(QueryUrlError)` - If the request or response processing fails
-    async fn query_url(api_key: &str, func_id: &str) -> Result<String> {
-        let client = reqwest::Client::new();
-
-        let uri = {
-            // вот бы добавили named-scopes как в котлине,
-            // чтоб мне не пришлось такой хуйнёй страдать.
-            #[allow(unused_assignments)]
-            let mut uri = String::new();
-            let mut counter = 0;
-
-            loop {
-                if counter == 3 {
-                    return Err(Error::EmptyUri);
-                }
-
-                counter += 1;
-
-                uri = client
-                    .post(format!(
-                        "https://functions.yandexcloud.net/{}?integration=raw",
-                        func_id
-                    ))
-                    .header("Authorization", format!("Api-Key {}", api_key))
-                    .send()
-                    .await
-                    .map_err(Error::Reqwest)?
-                    .text()
-                    .await
-                    .map_err(Error::Reqwest)?;
-
-                if uri.is_empty() {
-                    log::warn!("[{}] Unable to get uri! Retrying in 5 seconds...", counter);
-                    continue;
-                }
-
-                break;
-            }
-
-            uri
-        };
-
-        Ok(format!("https://politehnikum-eng.ru{}", uri.trim()))
-    }
-
-    /// Initializes the schedule by fetching the URL from the environment or Yandex Cloud Function (FaaS)
-    /// and creating a [`ScheduleSnapshot`] with the downloaded data.
-    ///
-    /// # Arguments
-    ///
-    /// * `downloader`: Mutable reference to an `XLSDownloader` implementation used to fetch and parse the schedule
-    /// * `app_env`: Reference to the application environment containing either a predefined URL or Yandex Cloud credentials
-    ///
-    /// # Returns
-    ///
-    /// Returns `Ok(())` if the snapshot was successfully initialized, or an `Error` if:
-    /// - URL query to Yandex Cloud failed ([`QueryUrlError`])
-    /// - Schedule snapshot creation failed ([`SnapshotCreationError`])
     pub async fn new(update_source: UpdateSource) -> Result<(Self, ScheduleSnapshot)> {
         let mut this = Updater {
-            downloader: XlsDownloader::new(),
             update_source,
+            version: None,
         };
 
         if let UpdateSource::Prepared(snapshot) = &this.update_source {
@@ -153,43 +95,24 @@ impl Updater {
             return Ok((this, snapshot));
         }
 
-        let url = match &this.update_source {
+        log::info!("Creating the initial schedule snapshot...");
-            UpdateSource::Url(url) => {
-                log::info!("The default link {} will be used", url);
-                url.clone()
-            }
-            UpdateSource::GrabFromSite {
-                yandex_api_key,
-                yandex_func_id,
-            } => {
-                log::info!("Obtaining a link using FaaS...");
-                Self::query_url(yandex_api_key, yandex_func_id).await?
-            }
-            _ => unreachable!(),
-        };
 
-        log::info!("For the initial setup, a link {} will be used", url);
+        let snapshot = this.new_snapshot().await?;
-
-        let snapshot = Self::new_snapshot(&mut this.downloader, url).await?;
         log::info!("Schedule snapshot successfully created!");
 
         Ok((this, snapshot))
     }
 
-    /// Updates the schedule snapshot by querying the latest URL from FaaS and checking for changes.
+    /// Rebuilds the schedule snapshot from the current remote file.
-    /// If the URL hasn't changed, only updates the [`fetched_at`] timestamp. If changed, downloads
+    ///
-    /// and parses the new schedule data.
+    /// When the remote file has not changed, the current snapshot is reused with a refreshed
+    /// fetch timestamp.
     ///
     /// # Arguments
     ///
-    /// * `downloader`: XLS file downloader used to fetch and parse the schedule data
+    /// * `current_snapshot`: Snapshot the provider currently serves.
-    /// * `app_env`: Application environment containing Yandex Cloud configuration and auto-update settings
     ///
-    /// returns: `Result<(), Error>` - Returns error if URL query fails or schedule parsing encounters issues
+    /// returns: `Result<ScheduleSnapshot, Error>`
-    ///
-    /// # Safety
-    ///
-    /// Use `unsafe` to access the initialized snapshot, guaranteed valid by prior `init()` call
     pub async fn update(
         &mut self,
         current_snapshot: &ScheduleSnapshot,
@@ -200,18 +123,9 @@ impl Updater {
             return Ok(snapshot);
         }
 
-        let url = match &self.update_source {
+        let snapshot = match self.new_snapshot().await {
-            UpdateSource::Url(url) => url.clone(),
-            UpdateSource::GrabFromSite {
-                yandex_api_key,
-                yandex_func_id,
-            } => Self::query_url(yandex_api_key.as_str(), yandex_func_id.as_str()).await?,
-            _ => unreachable!(),
-        };
-
-        let snapshot = match Self::new_snapshot(&mut self.downloader, url).await {
             Ok(snapshot) => snapshot,
-            Err(Error::SameETag) => {
+            Err(Error::NotModified) => {
                 let mut clone = current_snapshot.clone();
                 clone.update();
 

providers/provider-engels-polytechnic/src/xls_downloader.rs

@@ -1,253 +0,0 @@
-use chrono::{DateTime, Utc};
-use derive_more::{Display, Error};
-use std::mem::discriminant;
-use std::sync::Arc;
-use utoipa::ToSchema;
-
-/// XLS data retrieval errors.
-#[derive(Clone, Debug, ToSchema, Display, Error)]
-pub enum FetchError {
-    /// File url is not set.
-    #[display("The link to the timetable was not provided earlier.")]
-    NoUrlProvided,
-
-    /// Unknown error.
-    #[display("An unknown error occurred while downloading the file.")]
-    #[schema(value_type = String)]
-    Reqwest(Arc<reqwest::Error>),
-
-    /// Server returned a status code different from 200.
-    #[display("Server returned a status code {status_code}.")]
-    BadStatusCode { status_code: u16 },
-
-    /// The url leads to a file of a different type.
-    #[display("The link leads to a file of type '{content_type}'.")]
-    BadContentType { content_type: String },
-
-    /// Server doesn't return expected headers.
-    #[display("Server doesn't return expected header(s) '{expected_header}'.")]
-    BadHeaders { expected_header: String },
-}
-
-impl FetchError {
-    pub fn unknown(error: Arc<reqwest::Error>) -> Self {
-        Self::Reqwest(error)
-    }
-
-    pub fn bad_status_code(status_code: u16) -> Self {
-        Self::BadStatusCode { status_code }
-    }
-
-    pub fn bad_content_type(content_type: &str) -> Self {
-        Self::BadContentType {
-            content_type: content_type.to_string(),
-        }
-    }
-
-    pub fn bad_headers(expected_header: &str) -> Self {
-        Self::BadHeaders {
-            expected_header: expected_header.to_string(),
-        }
-    }
-}
-
-impl PartialEq for FetchError {
-    fn eq(&self, other: &Self) -> bool {
-        discriminant(self) == discriminant(other)
-    }
-}
-
-/// Result of XLS data retrieval.
-#[derive(Debug, PartialEq)]
-pub struct FetchOk {
-    /// File upload date.
-    pub uploaded_at: DateTime<Utc>,
-
-    /// Date data received.
-    pub requested_at: DateTime<Utc>,
-
-    /// Etag.
-    pub etag: String,
-
-    /// File data.
-    pub data: Option<Vec<u8>>,
-}
-
-impl FetchOk {
-    /// Result without file content.
-    pub fn head(uploaded_at: DateTime<Utc>, etag: String) -> Self {
-        FetchOk {
-            uploaded_at,
-            requested_at: Utc::now(),
-            etag,
-            data: None,
-        }
-    }
-
-    /// Full result.
-    pub fn get(uploaded_at: DateTime<Utc>, etag: String, data: Vec<u8>) -> Self {
-        FetchOk {
-            uploaded_at,
-            requested_at: Utc::now(),
-            etag,
-            data: Some(data),
-        }
-    }
-}
-
-pub type FetchResult = Result<FetchOk, FetchError>;
-
-pub struct XlsDownloader {
-    pub url: Option<String>,
-    pub etag: Option<String>,
-}
-
-impl XlsDownloader {
-    pub fn new() -> Self {
-        XlsDownloader {
-            url: None,
-            etag: None,
-        }
-    }
-
-    async fn fetch_specified(url: &str, head: bool) -> FetchResult {
-        let client = reqwest::Client::new();
-
-        let response = if head {
-            client.head(url)
-        } else {
-            client.get(url)
-        }
-        .header("User-Agent", ua_generator::ua::spoof_chrome_ua())
-        .send()
-        .await
-        .map_err(|e| FetchError::unknown(Arc::new(e)))?;
-
-        if response.status().as_u16() != 200 {
-            return Err(FetchError::bad_status_code(response.status().as_u16()));
-        }
-
-        let headers = response.headers();
-
-        let content_type = headers
-            .get("Content-Type")
-            .ok_or(FetchError::bad_headers("Content-Type"))?;
-
-        let etag = headers
-            .get("etag")
-            .ok_or(FetchError::bad_headers("etag"))?
-            .to_str()
-            .or(Err(FetchError::bad_headers("etag")))?
-            .to_string();
-
-        let last_modified = headers
-            .get("last-modified")
-            .ok_or(FetchError::bad_headers("last-modified"))?;
-
-        if content_type != "application/vnd.ms-excel" {
-            return Err(FetchError::bad_content_type(content_type.to_str().unwrap()));
-        }
-
-        let last_modified = DateTime::parse_from_rfc2822(last_modified.to_str().unwrap())
-            .unwrap()
-            .with_timezone(&Utc);
-
-        Ok(if head {
-            FetchOk::head(last_modified, etag)
-        } else {
-            FetchOk::get(
-                last_modified,
-                etag,
-                response.bytes().await.unwrap().to_vec(),
-            )
-        })
-    }
-
-    pub async fn fetch(&self, head: bool) -> FetchResult {
-        if self.url.is_none() {
-            Err(FetchError::NoUrlProvided)
-        } else {
-            Self::fetch_specified(self.url.as_ref().unwrap(), head).await
-        }
-    }
-
-    pub async fn set_url(&mut self, url: &str) -> FetchResult {
-        let result = Self::fetch_specified(url, true).await;
-
-        if result.is_ok() {
-            self.url = Some(url.to_string());
-        }
-
-        result
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::xls_downloader::{FetchError, XlsDownloader};
-
-    #[tokio::test]
-    async fn bad_url() {
-        let url = "bad_url";
-
-        let mut downloader = XlsDownloader::new();
-        assert!(downloader.set_url(url).await.is_err());
-    }
-
-    #[tokio::test]
-    async fn bad_status_code() {
-        let url = "https://www.google.com/not-found";
-
-        let mut downloader = XlsDownloader::new();
-        assert_eq!(
-            downloader.set_url(url).await,
-            Err(FetchError::bad_status_code(404))
-        );
-    }
-
-    #[tokio::test]
-    async fn bad_headers() {
-        let url = "https://www.google.com/favicon.ico";
-
-        let mut downloader = XlsDownloader::new();
-        assert_eq!(
-            downloader.set_url(url).await,
-            Err(FetchError::BadHeaders {
-                expected_header: "ETag".to_string(),
-            })
-        );
-    }
-
-    #[tokio::test]
-    async fn bad_content_type() {
-        let url = "https://s3.aero-storage.ldragol.ru/679e5d1145a6ad00843ad3f1/67ddb59fd46303008396ac96%2Fexample.txt";
-
-        let mut downloader = XlsDownloader::new();
-        assert!(downloader.set_url(url).await.is_err());
-    }
-
-    #[tokio::test]
-    async fn ok() {
-        let url = "https://s3.aero-storage.ldragol.ru/679e5d1145a6ad00843ad3f1/67ddb5fad46303008396ac97%2Fschedule.xls";
-
-        let mut downloader = XlsDownloader::new();
-        assert!(downloader.set_url(url).await.is_ok());
-    }
-
-    #[tokio::test]
-    async fn downloader_ok() {
-        let url = "https://s3.aero-storage.ldragol.ru/679e5d1145a6ad00843ad3f1/67ddb5fad46303008396ac97%2Fschedule.xls";
-
-        let mut downloader = XlsDownloader::new();
-        assert!(downloader.set_url(url).await.is_ok());
-        assert!(downloader.fetch(false).await.is_ok());
-    }
-
-    #[tokio::test]
-    async fn downloader_no_url_provided() {
-        let downloader = XlsDownloader::new();
-
-        let result = downloader.fetch(false).await;
-        assert_eq!(result, Err(FetchError::NoUrlProvided));
-    }
-}

providers/provider-engels-polytechnic/src/xls_downloader/mod.rs

@@ -0,0 +1,99 @@
+use chrono::{DateTime, Utc};
+use derive_more::{Display, Error};
+use std::mem::discriminant;
+use std::sync::Arc;
+
+mod yandex_disk;
+
+/// XLS data retrieval errors.
+#[derive(Clone, Debug, Display, Error)]
+pub enum FetchError {
+    /// Unknown error.
+    #[display("An unknown error occurred while downloading the file.")]
+    Reqwest(Arc<reqwest::Error>),
+
+    /// Server returned a status code different from 200.
+    #[display("Server returned a status code {status_code}.")]
+    BadStatusCode { status_code: u16 },
+
+    /// The folder contains no file matching the schedule name pattern.
+    #[display("No schedule file was found in the shared folder.")]
+    NoScheduleFile,
+}
+
+impl FetchError {
+    pub fn unknown(error: Arc<reqwest::Error>) -> Self {
+        Self::Reqwest(error)
+    }
+
+    pub fn bad_status_code(status_code: u16) -> Self {
+        Self::BadStatusCode { status_code }
+    }
+}
+
+impl PartialEq for FetchError {
+    fn eq(&self, other: &Self) -> bool {
+        discriminant(self) == discriminant(other)
+    }
+}
+
+pub type FetchResult<T> = Result<T, FetchError>;
+
+/// Description of the remote schedule file, obtained without downloading its content.
+#[derive(Clone, Debug, PartialEq)]
+pub struct RemoteFile {
+    /// Permanent link to the file, shown to API clients.
+    pub url: String,
+
+    /// Link the content is actually downloaded from.
+    pub download_url: String,
+
+    /// Content hash, changing whenever the file content changes.
+    pub version: String,
+
+    /// Time of the last file modification reported by the remote side.
+    pub modified_at: DateTime<Utc>,
+}
+
+/// Public Yandex Disk folder the schedule is downloaded from.
+#[derive(Clone, Debug)]
+pub struct Source {
+    pub public_url: String,
+}
+
+impl Source {
+    pub fn new(public_url: String) -> Self {
+        Self { public_url }
+    }
+
+    /// Looks up the current schedule file without downloading its content.
+    pub async fn probe(&self) -> FetchResult<RemoteFile> {
+        yandex_disk::probe(&self.public_url).await
+    }
+
+    /// Downloads the content of a previously probed file.
+    pub async fn download(&self, file: &RemoteFile) -> FetchResult<Vec<u8>> {
+        get(&file.download_url)
+            .await?
+            .bytes()
+            .await
+            .map(|bytes| bytes.to_vec())
+            .map_err(|error| FetchError::unknown(Arc::new(error)))
+    }
+}
+
+/// Performs a GET request with a spoofed browser User-Agent and checks the status code.
+async fn get(url: &str) -> FetchResult<reqwest::Response> {
+    let response = reqwest::Client::new()
+        .get(url)
+        .header("User-Agent", ua_generator::ua::spoof_chrome_ua())
+        .send()
+        .await
+        .map_err(|error| FetchError::unknown(Arc::new(error)))?;
+
+    if response.status().as_u16() != 200 {
+        return Err(FetchError::bad_status_code(response.status().as_u16()));
+    }
+
+    Ok(response)
+}

providers/provider-engels-polytechnic/src/xls_downloader/yandex_disk.rs

@@ -0,0 +1,135 @@
+use super::{FetchError, FetchResult, RemoteFile};
+use chrono::{DateTime, Utc};
+use percent_encoding::{AsciiSet, CONTROLS, NON_ALPHANUMERIC, utf8_percent_encode};
+use serde::Deserialize;
+
+/// Prefix of the schedule file names in the shared folder.
+const NAME_PREFIX: &str = "poltavskaja_";
+
+/// Marker of the corrections file, which holds a separate schedule.
+const NAME_EXCLUDED_MARKER: &str = "korr";
+
+/// Extension of the schedule files.
+const NAME_SUFFIX: &str = ".xls";
+
+/// Maximum amount of entries requested from the folder listing.
+const LISTING_LIMIT: u32 = 200;
+
+/// Characters not allowed inside a single path segment of the public file link.
+const PATH_SEGMENT: &AsciiSet = &CONTROLS
+    .add(b' ')
+    .add(b'"')
+    .add(b'#')
+    .add(b'%')
+    .add(b'/')
+    .add(b'<')
+    .add(b'>')
+    .add(b'?')
+    .add(b'`')
+    .add(b'{')
+    .add(b'}');
+
+#[derive(Deserialize)]
+struct Listing {
+    #[serde(rename = "_embedded")]
+    embedded: Embedded,
+}
+
+#[derive(Deserialize)]
+struct Embedded {
+    items: Vec<Item>,
+}
+
+#[derive(Deserialize)]
+struct Item {
+    #[serde(rename = "type")]
+    resource_type: String,
+    name: String,
+    modified: DateTime<Utc>,
+    md5: Option<String>,
+    revision: Option<u64>,
+    file: Option<String>,
+}
+
+impl Item {
+    /// Whether the entry is the schedule the provider is interested in.
+    fn is_schedule(&self) -> bool {
+        if self.resource_type != "file" || self.file.is_none() {
+            return false;
+        }
+
+        let name = self.name.to_lowercase();
+
+        name.starts_with(NAME_PREFIX)
+            && name.ends_with(NAME_SUFFIX)
+            && !name.contains(NAME_EXCLUDED_MARKER)
+    }
+
+    /// Marker changing whenever the file content changes.
+    fn version(&self) -> String {
+        self.md5
+            .clone()
+            .or_else(|| self.revision.map(|revision| revision.to_string()))
+            .unwrap_or_else(|| self.modified.to_rfc3339())
+    }
+}
+
+/// Finds the freshest schedule file in the public folder.
+///
+/// The files inside the folder are replaced independently of the folder link,
+/// so the whole listing is re-read on every probe.
+pub async fn probe(public_url: &str) -> FetchResult<RemoteFile> {
+    let listing = super::get(&format!(
+        "https://cloud-api.yandex.net/v1/disk/public/resources?public_key={}&limit={}",
+        utf8_percent_encode(public_url, NON_ALPHANUMERIC),
+        LISTING_LIMIT
+    ))
+    .await?
+    .json::<Listing>()
+    .await
+    .map_err(|error| FetchError::unknown(std::sync::Arc::new(error)))?;
+
+    let item = listing
+        .embedded
+        .items
+        .into_iter()
+        .filter(Item::is_schedule)
+        .max_by_key(|item| (item.modified, item.revision))
+        .ok_or(FetchError::NoScheduleFile)?;
+
+    Ok(RemoteFile {
+        url: format!(
+            "{}/{}",
+            public_url.trim_end_matches('/'),
+            utf8_percent_encode(&item.name, PATH_SEGMENT)
+        ),
+        version: item.version(),
+        modified_at: item.modified,
+        download_url: item.file.unwrap(),
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::probe;
+
+    const PUBLIC_URL: &str = "https://disk.yandex.ru/d/e8HJpMgDq7msyg";
+
+    #[tokio::test]
+    async fn probe_ok() {
+        let file = probe(PUBLIC_URL).await.unwrap();
+
+        assert!(file.url.starts_with(PUBLIC_URL));
+        assert!(!file.version.is_empty());
+        assert!(file.download_url.starts_with("https://"));
+    }
+
+    #[tokio::test]
+    async fn probe_unknown_folder() {
+        assert!(
+            probe("https://disk.yandex.ru/d/000000000000000")
+                .await
+                .is_err()
+        );
+    }
+}

src/state/env/mod.rs

@@ -2,22 +2,13 @@ pub mod schedule;
 pub mod telegram;
 pub mod vk_id;
 
-#[cfg(not(test))]
-pub mod yandex_cloud;
-
 pub use self::schedule::ScheduleEnvData;
 pub use self::telegram::TelegramEnvData;
 pub use self::vk_id::VkIdEnvData;
 
-#[cfg(not(test))]
-pub use self::yandex_cloud::YandexCloudEnvData;
-
 #[derive(Default)]
 pub struct AppEnv {
     pub schedule: ScheduleEnvData,
     pub telegram: TelegramEnvData,
     pub vk_id: VkIdEnvData,
-    
-    #[cfg(not(test))]
-    pub yandex_cloud: YandexCloudEnvData,
 }

src/state/env/schedule.rs

@@ -2,8 +2,10 @@ use std::env;
 
 #[derive(Clone)]
 pub struct ScheduleEnvData {
+    /// Public link to the Yandex Disk folder the schedule files are uploaded to.
     #[cfg(not(test))]
-    pub url: Option<String>,
+    pub yandex_disk_url: String,
+
     pub auto_update: bool,
 }
 
@@ -11,7 +13,8 @@ impl Default for ScheduleEnvData {
     fn default() -> Self {
         Self {
             #[cfg(not(test))]
-            url: env::var("SCHEDULE_INIT_URL").ok(),
+            yandex_disk_url: env::var("SCHEDULE_YANDEX_DISK_URL")
+                .expect("SCHEDULE_YANDEX_DISK_URL must be set"),
             auto_update: !env::var("SCHEDULE_DISABLE_AUTO_UPDATE")
                 .is_ok_and(|v| v.eq("1") || v.eq("true")),
         }

src/state/env/yandex_cloud.rs

@@ -1,16 +0,0 @@
-use std::env;
-
-#[derive(Clone)]
-pub struct YandexCloudEnvData {
-    pub api_key: String,
-    pub func_id: String,
-}
-
-impl Default for YandexCloudEnvData {
-    fn default() -> Self {
-        Self {
-            api_key: env::var("YANDEX_CLOUD_API_KEY").expect("YANDEX_CLOUD_API_KEY must be set"),
-            func_id: env::var("YANDEX_CLOUD_FUNC_ID").expect("YANDEX_CLOUD_FUNC_ID must be set"),
-        }
-    }
-}

src/state/mod.rs

@@ -38,13 +38,8 @@ impl AppState {
 
                 #[cfg(not(test))]
                 {
-                    if let Some(url) = &env.schedule.url {
+                    providers::EngelsPolytechnicUpdateSource::YandexDisk {
-                        providers::EngelsPolytechnicUpdateSource::Url(url.clone())
+                        public_url: env.schedule.yandex_disk_url.clone(),
-                    } else {
-                        providers::EngelsPolytechnicUpdateSource::GrabFromSite {
-                            yandex_api_key: env.yandex_cloud.api_key.clone(),
-                            yandex_func_id: env.yandex_cloud.func_id.clone(),
-                        }
                     }
                 }
             })