n08i40k/schedule-parser-rusted
1
0
Fork 0
mirror of https://github.com/n08i40k/schedule-parser-rusted.git synced 2026-09-14 08:58:53 +03:00
Code Issues Packages Projects Releases Wiki Activity

feat(downloader): download schedule from different source

Browse Source
This commit is contained in:
n08i40k
2026-09-04 22:54:18 +04:00
parent 3204c41fd5
commit 50a580b07c
15 changed files with 306 additions and 457 deletions
Download Patch File Download Diff File Expand all files Collapse all files
providers/provider-engels-polytechnic/src/lib.rs
-2
View File
@@ -63,8 +63,6 @@ impl ScheduleProvider for Wrapper {
this.snapshot = Arc::new(snapshot);
},
Err(updater::Error::EmptyUri) => {},
Err(err) => {
sentry::capture_error(&err);
}
providers/provider-engels-polytechnic/src/updater/error.rs
+6 -15
View File
@@ -3,25 +3,16 @@ use derive_more::{Display, Error, From};
#[derive(Debug, Display, Error, From)]
pub enum Error {
/// Occurs when the request to the Yandex Cloud API fails.
///
/// This may be due to network issues, invalid API key, incorrect function ID, or other
/// problems with the Yandex Cloud Function invocation.
#[display("An error occurred during the request to the Yandex Cloud API: {_0}")]
Reqwest(reqwest::Error),
/// The remote file has not changed since the last update.
#[display("The schedule file has not changed.")]
NotModified,
#[display("Unable to get URI in 3 retries")]
EmptyUri,
/// The ETag is the same (no update needed).
#[display("The ETag is the same.")]
SameETag,
/// The URL query for the XLS file failed to execute, either due to network issues or invalid API parameters.
/// The lookup of the current schedule file failed, either due to network issues or an
/// unexpected response from the storage.
#[display("Failed to fetch URL: {_0}")]
ScheduleFetchFailed(FetchError),
/// Downloading the XLS file content failed after successfully obtaining the URL.
/// Downloading the XLS file content failed after successfully locating the file.
#[display("Download failed: {_0}")]
ScheduleDownloadFailed(FetchError),
providers/provider-engels-polytechnic/src/updater/mod.rs
+54 -140
View File
@@ -1,40 +1,48 @@
pub use self::error::{Error, Result};
use crate::parser::parse_xls;
use crate::xls_downloader::{FetchError, XlsDownloader};
use crate::xls_downloader::{FetchError, Source};
use base::ScheduleSnapshot;
use chrono::Utc;
mod error;
pub enum UpdateSource {
Prepared(ScheduleSnapshot),
Url(String),
GrabFromSite {
yandex_api_key: String,
yandex_func_id: String,
/// Public Yandex Disk folder the college uploads the schedule to.
YandexDisk {
public_url: String,
},
}
pub struct Updater {
downloader: XlsDownloader,
update_source: UpdateSource,
/// Version of the file the current snapshot was built from.
version: Option<String>,
}
impl Updater {
/// Constructs a new `ScheduleSnapshot` by downloading and parsing schedule data from the specified URL.
/// Place the schedule is downloaded from, or [`None`] for a prepared snapshot.
fn source(&self) -> Option<Source> {
match &self.update_source {
UpdateSource::Prepared(_) => None,
UpdateSource::YandexDisk { public_url } => Some(Source::new(public_url.clone())),
}
}
/// Constructs a new [`ScheduleSnapshot`] by downloading and parsing the current schedule file.
///
/// This method first checks if the provided URL is the same as the one already configured in the downloader.
/// If different, it updates the downloader's URL, fetches the XLS content, parses it, and creates a snapshot.
/// Errors are returned for URL conflicts, network issues, download failures, or invalid data.
/// The file is looked up first, and its content is downloaded only when the version marker
/// differs from the one the current snapshot was built from.
///
/// # Arguments
/// # Returns
///
/// * `downloader`: A mutable reference to an `XLSDownloader` implementation used to fetch and parse the schedule data.
/// * `url`: The source URL pointing to the XLS file containing schedule data.
///
/// returns: Result<ScheduleSnapshot, SnapshotCreationError>
async fn new_snapshot(downloader: &mut XlsDownloader, url: String) -> Result<ScheduleSnapshot> {
let head_result = downloader.set_url(&url).await.map_err(|error| {
/// Returns [`Error::NotModified`] when the remote file has not changed since the last update,
/// or an error describing the failed download or parsing.
async fn new_snapshot(&mut self) -> Result<ScheduleSnapshot> {
let source = self.source().expect("a prepared snapshot has no source");
let file = source.probe().await.map_err(|error| {
if let FetchError::Reqwest(error) = &error {
sentry::capture_error(&error);
}
@@ -42,110 +50,44 @@ impl Updater {
Error::ScheduleFetchFailed(error)
})?;
if downloader.etag == Some(head_result.etag) {
return Err(Error::SameETag);
if self.version.as_deref() == Some(file.version.as_str()) {
return Err(Error::NotModified);
}
let xls_data = downloader
.fetch(false)
.await
.map_err(|error| {
if let FetchError::Reqwest(error) = &error {
sentry::capture_error(&error);
}
let xls_data = source.download(&file).await.map_err(|error| {
if let FetchError::Reqwest(error) = &error {
sentry::capture_error(&error);
}
Error::ScheduleDownloadFailed(error)
})?
.data
.unwrap();
Error::ScheduleDownloadFailed(error)
})?;
let parse_result = parse_xls(&xls_data)?;
self.version = Some(file.version);
Ok(ScheduleSnapshot {
fetched_at: head_result.requested_at,
updated_at: head_result.uploaded_at,
url,
fetched_at: Utc::now(),
updated_at: file.modified_at,
url: file.url,
data: parse_result,
})
}
/// Queries the Yandex Cloud Function (FaaS) to obtain a URL for the schedule file.
///
/// This sends a POST request to the specified Yandex Cloud Function endpoint,
/// using the provided API key for authentication. The returned URI is combined
/// with the "https://politehnikum-eng.ru" base domain to form the complete URL.
/// Initializes the schedule by downloading the current file from the configured source.
///
/// # Arguments
///
/// * `api_key` - Authentication token for Yandex Cloud API
/// * `func_id` - ID of the target Yandex Cloud Function to invoke
/// * `update_source`: Place the schedule is taken from.
///
/// # Returns
///
/// Result containing:
/// - `Ok(String)` - Complete URL constructed from the Function's response
/// - `Err(QueryUrlError)` - If the request or response processing fails
async fn query_url(api_key: &str, func_id: &str) -> Result<String> {
let client = reqwest::Client::new();
let uri = {
// вот бы добавили named-scopes как в котлине,
// чтоб мне не пришлось такой хуйнёй страдать.
#[allow(unused_assignments)]
let mut uri = String::new();
let mut counter = 0;
loop {
if counter == 3 {
return Err(Error::EmptyUri);
}
counter += 1;
uri = client
.post(format!(
"https://functions.yandexcloud.net/{}?integration=raw",
func_id
))
.header("Authorization", format!("Api-Key {}", api_key))
.send()
.await
.map_err(Error::Reqwest)?
.text()
.await
.map_err(Error::Reqwest)?;
if uri.is_empty() {
log::warn!("[{}] Unable to get uri! Retrying in 5 seconds...", counter);
continue;
}
break;
}
uri
};
Ok(format!("https://politehnikum-eng.ru{}", uri.trim()))
}
/// Initializes the schedule by fetching the URL from the environment or Yandex Cloud Function (FaaS)
/// and creating a [`ScheduleSnapshot`] with the downloaded data.
///
/// # Arguments
///
/// * `downloader`: Mutable reference to an `XLSDownloader` implementation used to fetch and parse the schedule
/// * `app_env`: Reference to the application environment containing either a predefined URL or Yandex Cloud credentials
///
/// # Returns
///
/// Returns `Ok(())` if the snapshot was successfully initialized, or an `Error` if:
/// - URL query to Yandex Cloud failed ([`QueryUrlError`])
/// - Schedule snapshot creation failed ([`SnapshotCreationError`])
/// Returns the updater together with the initial [`ScheduleSnapshot`], or an error if the
/// schedule could not be downloaded or parsed.
pub async fn new(update_source: UpdateSource) -> Result<(Self, ScheduleSnapshot)> {
let mut this = Updater {
downloader: XlsDownloader::new(),
update_source,
version: None,
};
if let UpdateSource::Prepared(snapshot) = &this.update_source {
@@ -153,43 +95,24 @@ impl Updater {
return Ok((this, snapshot));
}
let url = match &this.update_source {
UpdateSource::Url(url) => {
log::info!("The default link {} will be used", url);
url.clone()
}
UpdateSource::GrabFromSite {
yandex_api_key,
yandex_func_id,
} => {
log::info!("Obtaining a link using FaaS...");
Self::query_url(yandex_api_key, yandex_func_id).await?
}
_ => unreachable!(),
};
log::info!("Creating the initial schedule snapshot...");
log::info!("For the initial setup, a link {} will be used", url);
let snapshot = Self::new_snapshot(&mut this.downloader, url).await?;
let snapshot = this.new_snapshot().await?;
log::info!("Schedule snapshot successfully created!");
Ok((this, snapshot))
}
/// Updates the schedule snapshot by querying the latest URL from FaaS and checking for changes.
/// If the URL hasn't changed, only updates the [`fetched_at`] timestamp. If changed, downloads
/// and parses the new schedule data.
/// Rebuilds the schedule snapshot from the current remote file.
///
/// When the remote file has not changed, the current snapshot is reused with a refreshed
/// fetch timestamp.
///
/// # Arguments
///
/// * `downloader`: XLS file downloader used to fetch and parse the schedule data
/// * `app_env`: Application environment containing Yandex Cloud configuration and auto-update settings
/// * `current_snapshot`: Snapshot the provider currently serves.
///
/// returns: `Result<(), Error>` - Returns error if URL query fails or schedule parsing encounters issues
///
/// # Safety
///
/// Use `unsafe` to access the initialized snapshot, guaranteed valid by prior `init()` call
/// returns: `Result<ScheduleSnapshot, Error>`
pub async fn update(
&mut self,
current_snapshot: &ScheduleSnapshot,
@@ -200,18 +123,9 @@ impl Updater {
return Ok(snapshot);
}
let url = match &self.update_source {
UpdateSource::Url(url) => url.clone(),
UpdateSource::GrabFromSite {
yandex_api_key,
yandex_func_id,
} => Self::query_url(yandex_api_key.as_str(), yandex_func_id.as_str()).await?,
_ => unreachable!(),
};
let snapshot = match Self::new_snapshot(&mut self.downloader, url).await {
let snapshot = match self.new_snapshot().await {
Ok(snapshot) => snapshot,
Err(Error::SameETag) => {
Err(Error::NotModified) => {
let mut clone = current_snapshot.clone();
clone.update();
providers/provider-engels-polytechnic/src/xls_downloader.rs
-253
View File
@@ -1,253 +0,0 @@
use chrono::{DateTime, Utc};
use derive_more::{Display, Error};
use std::mem::discriminant;
use std::sync::Arc;
use utoipa::ToSchema;
/// XLS data retrieval errors.
#[derive(Clone, Debug, ToSchema, Display, Error)]
pub enum FetchError {
/// File url is not set.
#[display("The link to the timetable was not provided earlier.")]
NoUrlProvided,
/// Unknown error.
#[display("An unknown error occurred while downloading the file.")]
#[schema(value_type = String)]
Reqwest(Arc<reqwest::Error>),
/// Server returned a status code different from 200.
#[display("Server returned a status code {status_code}.")]
BadStatusCode { status_code: u16 },
/// The url leads to a file of a different type.
#[display("The link leads to a file of type '{content_type}'.")]
BadContentType { content_type: String },
/// Server doesn't return expected headers.
#[display("Server doesn't return expected header(s) '{expected_header}'.")]
BadHeaders { expected_header: String },
}
impl FetchError {
pub fn unknown(error: Arc<reqwest::Error>) -> Self {
Self::Reqwest(error)
}
pub fn bad_status_code(status_code: u16) -> Self {
Self::BadStatusCode { status_code }
}
pub fn bad_content_type(content_type: &str) -> Self {
Self::BadContentType {
content_type: content_type.to_string(),
}
}
pub fn bad_headers(expected_header: &str) -> Self {
Self::BadHeaders {
expected_header: expected_header.to_string(),
}
}
}
impl PartialEq for FetchError {
fn eq(&self, other: &Self) -> bool {
discriminant(self) == discriminant(other)
}
}
/// Result of XLS data retrieval.
#[derive(Debug, PartialEq)]
pub struct FetchOk {
/// File upload date.
pub uploaded_at: DateTime<Utc>,
/// Date data received.
pub requested_at: DateTime<Utc>,
/// Etag.
pub etag: String,
/// File data.
pub data: Option<Vec<u8>>,
}
impl FetchOk {
/// Result without file content.
pub fn head(uploaded_at: DateTime<Utc>, etag: String) -> Self {
FetchOk {
uploaded_at,
requested_at: Utc::now(),
etag,
data: None,
}
}
/// Full result.
pub fn get(uploaded_at: DateTime<Utc>, etag: String, data: Vec<u8>) -> Self {
FetchOk {
uploaded_at,
requested_at: Utc::now(),
etag,
data: Some(data),
}
}
}
pub type FetchResult = Result<FetchOk, FetchError>;
pub struct XlsDownloader {
pub url: Option<String>,
pub etag: Option<String>,
}
impl XlsDownloader {
pub fn new() -> Self {
XlsDownloader {
url: None,
etag: None,
}
}
async fn fetch_specified(url: &str, head: bool) -> FetchResult {
let client = reqwest::Client::new();
let response = if head {
client.head(url)
} else {
client.get(url)
}
.header("User-Agent", ua_generator::ua::spoof_chrome_ua())
.send()
.await
.map_err(|e| FetchError::unknown(Arc::new(e)))?;
if response.status().as_u16() != 200 {
return Err(FetchError::bad_status_code(response.status().as_u16()));
}
let headers = response.headers();
let content_type = headers
.get("Content-Type")
.ok_or(FetchError::bad_headers("Content-Type"))?;
let etag = headers
.get("etag")
.ok_or(FetchError::bad_headers("etag"))?
.to_str()
.or(Err(FetchError::bad_headers("etag")))?
.to_string();
let last_modified = headers
.get("last-modified")
.ok_or(FetchError::bad_headers("last-modified"))?;
if content_type != "application/vnd.ms-excel" {
return Err(FetchError::bad_content_type(content_type.to_str().unwrap()));
}
let last_modified = DateTime::parse_from_rfc2822(last_modified.to_str().unwrap())
.unwrap()
.with_timezone(&Utc);
Ok(if head {
FetchOk::head(last_modified, etag)
} else {
FetchOk::get(
last_modified,
etag,
response.bytes().await.unwrap().to_vec(),
)
})
}
pub async fn fetch(&self, head: bool) -> FetchResult {
if self.url.is_none() {
Err(FetchError::NoUrlProvided)
} else {
Self::fetch_specified(self.url.as_ref().unwrap(), head).await
}
}
pub async fn set_url(&mut self, url: &str) -> FetchResult {
let result = Self::fetch_specified(url, true).await;
if result.is_ok() {
self.url = Some(url.to_string());
}
result
}
}
#[cfg(test)]
mod tests {
use crate::xls_downloader::{FetchError, XlsDownloader};
#[tokio::test]
async fn bad_url() {
let url = "bad_url";
let mut downloader = XlsDownloader::new();
assert!(downloader.set_url(url).await.is_err());
}
#[tokio::test]
async fn bad_status_code() {
let url = "https://www.google.com/not-found";
let mut downloader = XlsDownloader::new();
assert_eq!(
downloader.set_url(url).await,
Err(FetchError::bad_status_code(404))
);
}
#[tokio::test]
async fn bad_headers() {
let url = "https://www.google.com/favicon.ico";
let mut downloader = XlsDownloader::new();
assert_eq!(
downloader.set_url(url).await,
Err(FetchError::BadHeaders {
expected_header: "ETag".to_string(),
})
);
}
#[tokio::test]
async fn bad_content_type() {
let url = "https://s3.aero-storage.ldragol.ru/679e5d1145a6ad00843ad3f1/67ddb59fd46303008396ac96%2Fexample.txt";
let mut downloader = XlsDownloader::new();
assert!(downloader.set_url(url).await.is_err());
}
#[tokio::test]
async fn ok() {
let url = "https://s3.aero-storage.ldragol.ru/679e5d1145a6ad00843ad3f1/67ddb5fad46303008396ac97%2Fschedule.xls";
let mut downloader = XlsDownloader::new();
assert!(downloader.set_url(url).await.is_ok());
}
#[tokio::test]
async fn downloader_ok() {
let url = "https://s3.aero-storage.ldragol.ru/679e5d1145a6ad00843ad3f1/67ddb5fad46303008396ac97%2Fschedule.xls";
let mut downloader = XlsDownloader::new();
assert!(downloader.set_url(url).await.is_ok());
assert!(downloader.fetch(false).await.is_ok());
}
#[tokio::test]
async fn downloader_no_url_provided() {
let downloader = XlsDownloader::new();
let result = downloader.fetch(false).await;
assert_eq!(result, Err(FetchError::NoUrlProvided));
}
}
providers/provider-engels-polytechnic/src/xls_downloader/mod.rs
+99
View File
@@ -0,0 +1,99 @@
use chrono::{DateTime, Utc};
use derive_more::{Display, Error};
use std::mem::discriminant;
use std::sync::Arc;
mod yandex_disk;
/// XLS data retrieval errors.
#[derive(Clone, Debug, Display, Error)]
pub enum FetchError {
/// Unknown error.
#[display("An unknown error occurred while downloading the file.")]
Reqwest(Arc<reqwest::Error>),
/// Server returned a status code different from 200.
#[display("Server returned a status code {status_code}.")]
BadStatusCode { status_code: u16 },
/// The folder contains no file matching the schedule name pattern.
#[display("No schedule file was found in the shared folder.")]
NoScheduleFile,
}
impl FetchError {
pub fn unknown(error: Arc<reqwest::Error>) -> Self {
Self::Reqwest(error)
}
pub fn bad_status_code(status_code: u16) -> Self {
Self::BadStatusCode { status_code }
}
}
impl PartialEq for FetchError {
fn eq(&self, other: &Self) -> bool {
discriminant(self) == discriminant(other)
}
}
pub type FetchResult<T> = Result<T, FetchError>;
/// Description of the remote schedule file, obtained without downloading its content.
#[derive(Clone, Debug, PartialEq)]
pub struct RemoteFile {
/// Permanent link to the file, shown to API clients.
pub url: String,
/// Link the content is actually downloaded from.
pub download_url: String,
/// Content hash, changing whenever the file content changes.
pub version: String,
/// Time of the last file modification reported by the remote side.
pub modified_at: DateTime<Utc>,
}
/// Public Yandex Disk folder the schedule is downloaded from.
#[derive(Clone, Debug)]
pub struct Source {
pub public_url: String,
}
impl Source {
pub fn new(public_url: String) -> Self {
Self { public_url }
}
/// Looks up the current schedule file without downloading its content.
pub async fn probe(&self) -> FetchResult<RemoteFile> {
yandex_disk::probe(&self.public_url).await
}
/// Downloads the content of a previously probed file.
pub async fn download(&self, file: &RemoteFile) -> FetchResult<Vec<u8>> {
get(&file.download_url)
.await?
.bytes()
.await
.map(|bytes| bytes.to_vec())
.map_err(|error| FetchError::unknown(Arc::new(error)))
}
}
/// Performs a GET request with a spoofed browser User-Agent and checks the status code.
async fn get(url: &str) -> FetchResult<reqwest::Response> {
let response = reqwest::Client::new()
.get(url)
.header("User-Agent", ua_generator::ua::spoof_chrome_ua())
.send()
.await
.map_err(|error| FetchError::unknown(Arc::new(error)))?;
if response.status().as_u16() != 200 {
return Err(FetchError::bad_status_code(response.status().as_u16()));
}
Ok(response)
}
providers/provider-engels-polytechnic/src/xls_downloader/yandex_disk.rs
+135
View File
@@ -0,0 +1,135 @@
use super::{FetchError, FetchResult, RemoteFile};
use chrono::{DateTime, Utc};
use percent_encoding::{AsciiSet, CONTROLS, NON_ALPHANUMERIC, utf8_percent_encode};
use serde::Deserialize;
/// Prefix of the schedule file names in the shared folder.
const NAME_PREFIX: &str = "poltavskaja_";
/// Marker of the corrections file, which holds a separate schedule.
const NAME_EXCLUDED_MARKER: &str = "korr";
/// Extension of the schedule files.
const NAME_SUFFIX: &str = ".xls";
/// Maximum amount of entries requested from the folder listing.
const LISTING_LIMIT: u32 = 200;
/// Characters not allowed inside a single path segment of the public file link.
const PATH_SEGMENT: &AsciiSet = &CONTROLS
.add(b' ')
.add(b'"')
.add(b'#')
.add(b'%')
.add(b'/')
.add(b'<')
.add(b'>')
.add(b'?')
.add(b'`')
.add(b'{')
.add(b'}');
#[derive(Deserialize)]
struct Listing {
#[serde(rename = "_embedded")]
embedded: Embedded,
}
#[derive(Deserialize)]
struct Embedded {
items: Vec<Item>,
}
#[derive(Deserialize)]
struct Item {
#[serde(rename = "type")]
resource_type: String,
name: String,
modified: DateTime<Utc>,
md5: Option<String>,
revision: Option<u64>,
file: Option<String>,
}
impl Item {
/// Whether the entry is the schedule the provider is interested in.
fn is_schedule(&self) -> bool {
if self.resource_type != "file" || self.file.is_none() {
return false;
}
let name = self.name.to_lowercase();
name.starts_with(NAME_PREFIX)
&& name.ends_with(NAME_SUFFIX)
&& !name.contains(NAME_EXCLUDED_MARKER)
}
/// Marker changing whenever the file content changes.
fn version(&self) -> String {
self.md5
.clone()
.or_else(|| self.revision.map(|revision| revision.to_string()))
.unwrap_or_else(|| self.modified.to_rfc3339())
}
}
/// Finds the freshest schedule file in the public folder.
///
/// The files inside the folder are replaced independently of the folder link,
/// so the whole listing is re-read on every probe.
pub async fn probe(public_url: &str) -> FetchResult<RemoteFile> {
let listing = super::get(&format!(
"https://cloud-api.yandex.net/v1/disk/public/resources?public_key={}&limit={}",
utf8_percent_encode(public_url, NON_ALPHANUMERIC),
LISTING_LIMIT
))
.await?
.json::<Listing>()
.await
.map_err(|error| FetchError::unknown(std::sync::Arc::new(error)))?;
let item = listing
.embedded
.items
.into_iter()
.filter(Item::is_schedule)
.max_by_key(|item| (item.modified, item.revision))
.ok_or(FetchError::NoScheduleFile)?;
Ok(RemoteFile {
url: format!(
"{}/{}",
public_url.trim_end_matches('/'),
utf8_percent_encode(&item.name, PATH_SEGMENT)
),
version: item.version(),
modified_at: item.modified,
download_url: item.file.unwrap(),
})
}
#[cfg(test)]
mod tests {
use super::probe;
const PUBLIC_URL: &str = "https://disk.yandex.ru/d/e8HJpMgDq7msyg";
#[tokio::test]
async fn probe_ok() {
let file = probe(PUBLIC_URL).await.unwrap();
assert!(file.url.starts_with(PUBLIC_URL));
assert!(!file.version.is_empty());
assert!(file.download_url.starts_with("https://"));
}
#[tokio::test]
async fn probe_unknown_folder() {
assert!(
probe("https://disk.yandex.ru/d/000000000000000")
.await
.is_err()
);
}
}