Compare commits

..
25 changed files with 2282 additions and 2134 deletions
+1 -5
View File
@@ -1,5 +1,5 @@
# Schedule # Schedule
# SCHEDULE_INIT_URL= SCHEDULE_YANDEX_DISK_URL="https://disk.yandex.ru/d/xxxxxxxxxxxxxx"
SCHEDULE_DISABLE_AUTO_UPDATE=1 SCHEDULE_DISABLE_AUTO_UPDATE=1
# Basic authorization # Basic authorization
@@ -14,10 +14,6 @@ TELEGRAM_BOT_ID=0
TELEGRAM_MINI_APP_HOST=example.com TELEGRAM_MINI_APP_HOST=example.com
TELEGRAM_TEST_DC=false TELEGRAM_TEST_DC=false
# Yandex Cloud
YANDEX_CLOUD_API_KEY=""
YANDEX_CLOUD_FUNC_ID=""
# Firebase # Firebase
# GOOGLE_APPLICATION_CREDENTIALS= # GOOGLE_APPLICATION_CREDENTIALS=
-21
View File
@@ -13,12 +13,6 @@ env:
BINARY_NAME: schedule-parser-rusted BINARY_NAME: schedule-parser-rusted
TEST_DB: ${{ secrets.TEST_DATABASE_URL }}
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
DOCKER_IMAGE_NAME: ${{ github.repository }} DOCKER_IMAGE_NAME: ${{ github.repository }}
DOCKER_REGISTRY_HOST: registry.n08i40k.ru DOCKER_REGISTRY_HOST: registry.n08i40k.ru
@@ -43,7 +37,6 @@ jobs:
run: | run: |
cargo test cargo test
env: env:
DATABASE_URL: ${{ env.TEST_DB }}
SCHEDULE_DISABLE_AUTO_UPDATE: 1 SCHEDULE_DISABLE_AUTO_UPDATE: 1
JWT_SECRET: "test-secret-at-least-256-bits-used" JWT_SECRET: "test-secret-at-least-256-bits-used"
VK_ID_CLIENT_ID: 0 VK_ID_CLIENT_ID: 0
@@ -51,8 +44,6 @@ jobs:
TELEGRAM_BOT_ID: 0 TELEGRAM_BOT_ID: 0
TELEGRAM_MINI_APP_HOST: example.com TELEGRAM_MINI_APP_HOST: example.com
TELEGRAM_TEST_DC: false TELEGRAM_TEST_DC: false
YANDEX_CLOUD_API_KEY: ""
YANDEX_CLOUD_FUNC_ID: ""
build: build:
name: Build name: Build
runs-on: ubuntu-latest runs-on: ubuntu-latest
@@ -76,18 +67,6 @@ jobs:
objcopy --strip-debug --strip-unneeded target/release/${{ env.BINARY_NAME }} objcopy --strip-debug --strip-unneeded target/release/${{ env.BINARY_NAME }}
objcopy --add-gnu-debuglink target/release/${{ env.BINARY_NAME }}{.d,} objcopy --add-gnu-debuglink target/release/${{ env.BINARY_NAME }}{.d,}
- name: Setup sentry-cli
uses: matbour/setup-sentry-cli@v2.0.0
with:
version: latest
token: ${{ env.SENTRY_AUTH_TOKEN }}
organization: ${{ env.SENTRY_ORG }}
project: ${{ env.SENTRY_PROJECT }}
- name: Upload debug symbols to Sentry
run: |
sentry-cli debug-files upload --include-sources .
- name: Upload build binary artifact - name: Upload build binary artifact
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
-18
View File
@@ -14,10 +14,6 @@ env:
TEST_DB: ${{ secrets.TEST_DATABASE_URL }} TEST_DB: ${{ secrets.TEST_DATABASE_URL }}
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
DOCKER_IMAGE_NAME: ${{ github.repository }} DOCKER_IMAGE_NAME: ${{ github.repository }}
DOCKER_REGISTRY_HOST: registry.n08i40k.ru DOCKER_REGISTRY_HOST: registry.n08i40k.ru
@@ -50,8 +46,6 @@ jobs:
TELEGRAM_BOT_ID: 0 TELEGRAM_BOT_ID: 0
TELEGRAM_MINI_APP_HOST: example.com TELEGRAM_MINI_APP_HOST: example.com
TELEGRAM_TEST_DC: false TELEGRAM_TEST_DC: false
YANDEX_CLOUD_API_KEY: ""
YANDEX_CLOUD_FUNC_ID: ""
build: build:
name: Build name: Build
runs-on: ubuntu-latest runs-on: ubuntu-latest
@@ -75,18 +69,6 @@ jobs:
objcopy --strip-debug --strip-unneeded target/release/${{ env.BINARY_NAME }} objcopy --strip-debug --strip-unneeded target/release/${{ env.BINARY_NAME }}
objcopy --add-gnu-debuglink target/release/${{ env.BINARY_NAME }}{.d,} objcopy --add-gnu-debuglink target/release/${{ env.BINARY_NAME }}{.d,}
- name: Setup sentry-cli
uses: matbour/setup-sentry-cli@v2.0.0
with:
version: latest
token: ${{ env.SENTRY_AUTH_TOKEN }}
organization: ${{ env.SENTRY_ORG }}
project: ${{ env.SENTRY_PROJECT }}
- name: Upload debug symbols to Sentry
run: |
sentry-cli debug-files upload --include-sources .
- name: Upload build binary artifact - name: Upload build binary artifact
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
-2
View File
@@ -33,5 +33,3 @@ jobs:
TELEGRAM_BOT_ID: 0 TELEGRAM_BOT_ID: 0
TELEGRAM_MINI_APP_HOST: example.com TELEGRAM_MINI_APP_HOST: example.com
TELEGRAM_TEST_DC: false TELEGRAM_TEST_DC: false
YANDEX_CLOUD_API_KEY: ""
YANDEX_CLOUD_FUNC_ID: ""
Generated
+1850 -1585
View File
File diff suppressed because it is too large Load Diff
+15 -4
View File
@@ -38,17 +38,27 @@ futures-util = "0"
# authorization # authorization
bcrypt = "0" bcrypt = "0"
jsonwebtoken = { version = "9", features = ["use_pem"] } jsonwebtoken = { version = "11", features = ["use_pem", "aws_lc_rs"] }
# creating users # creating users
objectid = "0" objectid = "0"
# schedule downloader # schedule downloader
reqwest = { version = "0", features = ["json"] } reqwest = { version = "0", features = ["json", "form"] }
mime = "0" mime = "0"
# error handling # error handling
sentry = "0" sentry = { version = "0", default-features = false, features = [
"backtrace",
"contexts",
"debug-images",
"logs",
"metrics",
"panic",
"release-health",
"reqwest",
"rustls",
] }
sentry-actix = "0" sentry-actix = "0"
# [de]serializing # [de]serializing
@@ -70,7 +80,7 @@ log = "0"
# telegram webdata deciding and verify # telegram webdata deciding and verify
base64 = "0" base64 = "0"
percent-encoding = "2" percent-encoding = "2"
ed25519-dalek = "3.0.0-pre.1" aws-lc-rs = "1"
# development tracing # development tracing
console-subscriber = { version = "0", optional = true } console-subscriber = { version = "0", optional = true }
@@ -78,4 +88,5 @@ tracing = { version = "0", optional = true }
[dev-dependencies] [dev-dependencies]
providers = { path = "providers", features = ["test"] } providers = { path = "providers", features = ["test"] }
database = { path = "database", features = ["sqlite"] }
actix-test = { path = "actix-test" } actix-test = { path = "actix-test" }
+1 -1
View File
@@ -6,7 +6,7 @@ ARG BINARY_NAME
WORKDIR /app/ WORKDIR /app/
RUN apt update && \ RUN apt update && \
apt install -y libpq5 ca-certificates openssl apt install -y ca-certificates
COPY ./${BINARY_NAME} /bin/main COPY ./${BINARY_NAME} /bin/main
RUN chmod +x /bin/main RUN chmod +x /bin/main
+3
View File
@@ -3,6 +3,9 @@ name = "database"
version = "0.1.0" version = "0.1.0"
edition = "2024" edition = "2024"
[features]
sqlite = ["sea-orm/sqlx-sqlite", "migration/sqlite"]
[dependencies] [dependencies]
migration = { path = "migration" } migration = { path = "migration" }
entity = { path = "entity" } entity = { path = "entity" }
+3
View File
@@ -8,6 +8,9 @@ publish = false
name = "migration" name = "migration"
path = "src/lib.rs" path = "src/lib.rs"
[features]
sqlite = ["sea-orm-migration/sqlx-sqlite"]
[dependencies] [dependencies]
async-std = { version = "1", features = ["attributes", "tokio1"] } async-std = { version = "1", features = ["attributes", "tokio1"] }
+18 -12
View File
@@ -1,5 +1,5 @@
use sea_orm_migration::prelude::extension::postgres::Type; use sea_orm_migration::prelude::extension::postgres::Type;
use sea_orm_migration::sea_orm::{EnumIter, Iterable}; use sea_orm_migration::sea_orm::{DatabaseBackend, EnumIter, Iterable};
use sea_orm_migration::{prelude::*, schema::*}; use sea_orm_migration::{prelude::*, schema::*};
#[derive(DeriveMigrationName)] #[derive(DeriveMigrationName)]
@@ -8,14 +8,16 @@ pub struct Migration;
#[async_trait::async_trait] #[async_trait::async_trait]
impl MigrationTrait for Migration { impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager if manager.get_database_backend() == DatabaseBackend::Postgres {
.create_type( manager
Type::create() .create_type(
.as_enum(UserRole) Type::create()
.values(UserRoleVariants::iter()) .as_enum(UserRole)
.to_owned(), .values(UserRoleVariants::iter())
) .to_owned(),
.await?; )
.await?;
}
manager manager
.create_table( .create_table(
@@ -40,9 +42,13 @@ impl MigrationTrait for Migration {
.drop_table(Table::drop().table(User::Table).to_owned()) .drop_table(Table::drop().table(User::Table).to_owned())
.await?; .await?;
manager if manager.get_database_backend() == DatabaseBackend::Postgres {
.drop_type(Type::drop().name(UserRole).to_owned()) manager
.await .drop_type(Type::drop().name(UserRole).to_owned())
.await?;
}
Ok(())
} }
} }
@@ -21,11 +21,23 @@ utoipa = { version = "5", features = ["macros", "chrono"] }
calamine = "0" calamine = "0"
async-trait = "0" async-trait = "0"
reqwest = "0" reqwest = { version = "0", features = ["json"] }
serde = { version = "1", features = ["derive"] }
percent-encoding = "2"
ua_generator = "0" ua_generator = "0"
regex = "1" regex = "1"
strsim = "0" strsim = "0"
log = "0" log = "0"
sentry = "0" sentry = { version = "0", default-features = false, features = [
"backtrace",
"contexts",
"debug-images",
"logs",
"metrics",
"panic",
"release-health",
"reqwest",
"rustls",
] }
fancy-regex = "0" fancy-regex = "0"
@@ -63,8 +63,6 @@ impl ScheduleProvider for Wrapper {
this.snapshot = Arc::new(snapshot); this.snapshot = Arc::new(snapshot);
}, },
Err(updater::Error::EmptyUri) => {},
Err(err) => { Err(err) => {
sentry::capture_error(&err); sentry::capture_error(&err);
} }
@@ -1,12 +1,12 @@
pub use self::error::{Error, Result}; pub use self::error::{Error, Result};
use crate::or_continue; use crate::or_continue;
use crate::parser::worksheet::{CellPos, CellRange, WorkSheet};
use crate::parser::LessonParseResult::{Lessons, Street}; use crate::parser::LessonParseResult::{Lessons, Street};
use crate::parser::worksheet::{CellPos, CellRange, WorkSheet};
use base::LessonType::Break; use base::LessonType::Break;
use base::{ use base::{
Day, Lesson, LessonBoundaries, LessonSubGroup, LessonType, ParsedSchedule, ScheduleEntry, Day, Lesson, LessonBoundaries, LessonSubGroup, LessonType, ParsedSchedule, ScheduleEntry,
}; };
use calamine::{open_workbook_from_rs, Reader, Xls}; use calamine::{Reader, Xls, open_workbook_from_rs};
use chrono::{DateTime, Duration, NaiveDate, NaiveTime, Utc}; use chrono::{DateTime, Duration, NaiveDate, NaiveTime, Utc};
use regex::Regex; use regex::Regex;
use std::collections::HashMap; use std::collections::HashMap;
@@ -243,12 +243,9 @@ fn parse_lesson(
.first() .first()
.ok_or(Error::LessonTimeNotFound(CellPos::new(row, group_column)))?; .ok_or(Error::LessonTimeNotFound(CellPos::new(row, group_column)))?;
let range: Option<[u8; 2]> = if lesson_boundaries.default_index.is_some() { let range: Option<[u8; 2]> = lesson_boundaries
let default = lesson_boundaries.default_index.unwrap() as u8; .default_index
Some([default, end_time.default_index.unwrap() as u8]) .map(|default_index| [default_index as u8, end_time.default_index.unwrap() as u8]);
} else {
None
};
let time = LessonBoundaries { let time = LessonBoundaries {
start: lesson_boundaries.time_range.start, start: lesson_boundaries.time_range.start,
@@ -743,8 +740,8 @@ pub fn parse_xls(buffer: &Vec<u8>) -> Result<ParsedSchedule> {
.clone(); .clone();
let worksheet_merges = workbook let worksheet_merges = workbook
.worksheet_merge_cells(&worksheet_name) .merge_cells_by_sheet_name(&worksheet_name)
.ok_or(Error::NoWorkSheets)?; .map_err(|_| Error::NoWorkSheets)?;
WorkSheet { WorkSheet {
data: worksheet, data: worksheet,
@@ -3,25 +3,16 @@ use derive_more::{Display, Error, From};
#[derive(Debug, Display, Error, From)] #[derive(Debug, Display, Error, From)]
pub enum Error { pub enum Error {
/// Occurs when the request to the Yandex Cloud API fails. /// The remote file has not changed since the last update.
/// #[display("The schedule file has not changed.")]
/// This may be due to network issues, invalid API key, incorrect function ID, or other NotModified,
/// problems with the Yandex Cloud Function invocation.
#[display("An error occurred during the request to the Yandex Cloud API: {_0}")]
Reqwest(reqwest::Error),
#[display("Unable to get URI in 3 retries")] /// The lookup of the current schedule file failed, either due to network issues or an
EmptyUri, /// unexpected response from the storage.
/// The ETag is the same (no update needed).
#[display("The ETag is the same.")]
SameETag,
/// The URL query for the XLS file failed to execute, either due to network issues or invalid API parameters.
#[display("Failed to fetch URL: {_0}")] #[display("Failed to fetch URL: {_0}")]
ScheduleFetchFailed(FetchError), ScheduleFetchFailed(FetchError),
/// Downloading the XLS file content failed after successfully obtaining the URL. /// Downloading the XLS file content failed after successfully locating the file.
#[display("Download failed: {_0}")] #[display("Download failed: {_0}")]
ScheduleDownloadFailed(FetchError), ScheduleDownloadFailed(FetchError),
@@ -1,40 +1,48 @@
pub use self::error::{Error, Result}; pub use self::error::{Error, Result};
use crate::parser::parse_xls; use crate::parser::parse_xls;
use crate::xls_downloader::{FetchError, XlsDownloader}; use crate::xls_downloader::{FetchError, Source};
use base::ScheduleSnapshot; use base::ScheduleSnapshot;
use chrono::Utc;
mod error; mod error;
pub enum UpdateSource { pub enum UpdateSource {
Prepared(ScheduleSnapshot), Prepared(ScheduleSnapshot),
Url(String), /// Public Yandex Disk folder the college uploads the schedule to.
YandexDisk {
GrabFromSite { public_url: String,
yandex_api_key: String,
yandex_func_id: String,
}, },
} }
pub struct Updater { pub struct Updater {
downloader: XlsDownloader,
update_source: UpdateSource, update_source: UpdateSource,
/// Version of the file the current snapshot was built from.
version: Option<String>,
} }
impl Updater { impl Updater {
/// Constructs a new `ScheduleSnapshot` by downloading and parsing schedule data from the specified URL. /// Place the schedule is downloaded from, or [`None`] for a prepared snapshot.
fn source(&self) -> Option<Source> {
match &self.update_source {
UpdateSource::Prepared(_) => None,
UpdateSource::YandexDisk { public_url } => Some(Source::new(public_url.clone())),
}
}
/// Constructs a new [`ScheduleSnapshot`] by downloading and parsing the current schedule file.
/// ///
/// This method first checks if the provided URL is the same as the one already configured in the downloader. /// The file is looked up first, and its content is downloaded only when the version marker
/// If different, it updates the downloader's URL, fetches the XLS content, parses it, and creates a snapshot. /// differs from the one the current snapshot was built from.
/// Errors are returned for URL conflicts, network issues, download failures, or invalid data.
/// ///
/// # Arguments /// # Returns
/// ///
/// * `downloader`: A mutable reference to an `XLSDownloader` implementation used to fetch and parse the schedule data. /// Returns [`Error::NotModified`] when the remote file has not changed since the last update,
/// * `url`: The source URL pointing to the XLS file containing schedule data. /// or an error describing the failed download or parsing.
/// async fn new_snapshot(&mut self) -> Result<ScheduleSnapshot> {
/// returns: Result<ScheduleSnapshot, SnapshotCreationError> let source = self.source().expect("a prepared snapshot has no source");
async fn new_snapshot(downloader: &mut XlsDownloader, url: String) -> Result<ScheduleSnapshot> {
let head_result = downloader.set_url(&url).await.map_err(|error| { let file = source.probe().await.map_err(|error| {
if let FetchError::Reqwest(error) = &error { if let FetchError::Reqwest(error) = &error {
sentry::capture_error(&error); sentry::capture_error(&error);
} }
@@ -42,110 +50,44 @@ impl Updater {
Error::ScheduleFetchFailed(error) Error::ScheduleFetchFailed(error)
})?; })?;
if downloader.etag == Some(head_result.etag) { if self.version.as_deref() == Some(file.version.as_str()) {
return Err(Error::SameETag); return Err(Error::NotModified);
} }
let xls_data = downloader let xls_data = source.download(&file).await.map_err(|error| {
.fetch(false) if let FetchError::Reqwest(error) = &error {
.await sentry::capture_error(&error);
.map_err(|error| { }
if let FetchError::Reqwest(error) = &error {
sentry::capture_error(&error);
}
Error::ScheduleDownloadFailed(error) Error::ScheduleDownloadFailed(error)
})? })?;
.data
.unwrap();
let parse_result = parse_xls(&xls_data)?; let parse_result = parse_xls(&xls_data)?;
self.version = Some(file.version);
Ok(ScheduleSnapshot { Ok(ScheduleSnapshot {
fetched_at: head_result.requested_at, fetched_at: Utc::now(),
updated_at: head_result.uploaded_at, updated_at: file.modified_at,
url, url: file.url,
data: parse_result, data: parse_result,
}) })
} }
/// Queries the Yandex Cloud Function (FaaS) to obtain a URL for the schedule file. /// Initializes the schedule by downloading the current file from the configured source.
///
/// This sends a POST request to the specified Yandex Cloud Function endpoint,
/// using the provided API key for authentication. The returned URI is combined
/// with the "https://politehnikum-eng.ru" base domain to form the complete URL.
/// ///
/// # Arguments /// # Arguments
/// ///
/// * `api_key` - Authentication token for Yandex Cloud API /// * `update_source`: Place the schedule is taken from.
/// * `func_id` - ID of the target Yandex Cloud Function to invoke
/// ///
/// # Returns /// # Returns
/// ///
/// Result containing: /// Returns the updater together with the initial [`ScheduleSnapshot`], or an error if the
/// - `Ok(String)` - Complete URL constructed from the Function's response /// schedule could not be downloaded or parsed.
/// - `Err(QueryUrlError)` - If the request or response processing fails
async fn query_url(api_key: &str, func_id: &str) -> Result<String> {
let client = reqwest::Client::new();
let uri = {
// вот бы добавили named-scopes как в котлине,
// чтоб мне не пришлось такой хуйнёй страдать.
#[allow(unused_assignments)]
let mut uri = String::new();
let mut counter = 0;
loop {
if counter == 3 {
return Err(Error::EmptyUri);
}
counter += 1;
uri = client
.post(format!(
"https://functions.yandexcloud.net/{}?integration=raw",
func_id
))
.header("Authorization", format!("Api-Key {}", api_key))
.send()
.await
.map_err(Error::Reqwest)?
.text()
.await
.map_err(Error::Reqwest)?;
if uri.is_empty() {
log::warn!("[{}] Unable to get uri! Retrying in 5 seconds...", counter);
continue;
}
break;
}
uri
};
Ok(format!("https://politehnikum-eng.ru{}", uri.trim()))
}
/// Initializes the schedule by fetching the URL from the environment or Yandex Cloud Function (FaaS)
/// and creating a [`ScheduleSnapshot`] with the downloaded data.
///
/// # Arguments
///
/// * `downloader`: Mutable reference to an `XLSDownloader` implementation used to fetch and parse the schedule
/// * `app_env`: Reference to the application environment containing either a predefined URL or Yandex Cloud credentials
///
/// # Returns
///
/// Returns `Ok(())` if the snapshot was successfully initialized, or an `Error` if:
/// - URL query to Yandex Cloud failed ([`QueryUrlError`])
/// - Schedule snapshot creation failed ([`SnapshotCreationError`])
pub async fn new(update_source: UpdateSource) -> Result<(Self, ScheduleSnapshot)> { pub async fn new(update_source: UpdateSource) -> Result<(Self, ScheduleSnapshot)> {
let mut this = Updater { let mut this = Updater {
downloader: XlsDownloader::new(),
update_source, update_source,
version: None,
}; };
if let UpdateSource::Prepared(snapshot) = &this.update_source { if let UpdateSource::Prepared(snapshot) = &this.update_source {
@@ -153,43 +95,24 @@ impl Updater {
return Ok((this, snapshot)); return Ok((this, snapshot));
} }
let url = match &this.update_source { log::info!("Creating the initial schedule snapshot...");
UpdateSource::Url(url) => {
log::info!("The default link {} will be used", url);
url.clone()
}
UpdateSource::GrabFromSite {
yandex_api_key,
yandex_func_id,
} => {
log::info!("Obtaining a link using FaaS...");
Self::query_url(yandex_api_key, yandex_func_id).await?
}
_ => unreachable!(),
};
log::info!("For the initial setup, a link {} will be used", url); let snapshot = this.new_snapshot().await?;
let snapshot = Self::new_snapshot(&mut this.downloader, url).await?;
log::info!("Schedule snapshot successfully created!"); log::info!("Schedule snapshot successfully created!");
Ok((this, snapshot)) Ok((this, snapshot))
} }
/// Updates the schedule snapshot by querying the latest URL from FaaS and checking for changes. /// Rebuilds the schedule snapshot from the current remote file.
/// If the URL hasn't changed, only updates the [`fetched_at`] timestamp. If changed, downloads ///
/// and parses the new schedule data. /// When the remote file has not changed, the current snapshot is reused with a refreshed
/// fetch timestamp.
/// ///
/// # Arguments /// # Arguments
/// ///
/// * `downloader`: XLS file downloader used to fetch and parse the schedule data /// * `current_snapshot`: Snapshot the provider currently serves.
/// * `app_env`: Application environment containing Yandex Cloud configuration and auto-update settings
/// ///
/// returns: `Result<(), Error>` - Returns error if URL query fails or schedule parsing encounters issues /// returns: `Result<ScheduleSnapshot, Error>`
///
/// # Safety
///
/// Use `unsafe` to access the initialized snapshot, guaranteed valid by prior `init()` call
pub async fn update( pub async fn update(
&mut self, &mut self,
current_snapshot: &ScheduleSnapshot, current_snapshot: &ScheduleSnapshot,
@@ -200,18 +123,9 @@ impl Updater {
return Ok(snapshot); return Ok(snapshot);
} }
let url = match &self.update_source { let snapshot = match self.new_snapshot().await {
UpdateSource::Url(url) => url.clone(),
UpdateSource::GrabFromSite {
yandex_api_key,
yandex_func_id,
} => Self::query_url(yandex_api_key.as_str(), yandex_func_id.as_str()).await?,
_ => unreachable!(),
};
let snapshot = match Self::new_snapshot(&mut self.downloader, url).await {
Ok(snapshot) => snapshot, Ok(snapshot) => snapshot,
Err(Error::SameETag) => { Err(Error::NotModified) => {
let mut clone = current_snapshot.clone(); let mut clone = current_snapshot.clone();
clone.update(); clone.update();
@@ -1,253 +0,0 @@
use chrono::{DateTime, Utc};
use derive_more::{Display, Error};
use std::mem::discriminant;
use std::sync::Arc;
use utoipa::ToSchema;
/// XLS data retrieval errors.
#[derive(Clone, Debug, ToSchema, Display, Error)]
pub enum FetchError {
/// File url is not set.
#[display("The link to the timetable was not provided earlier.")]
NoUrlProvided,
/// Unknown error.
#[display("An unknown error occurred while downloading the file.")]
#[schema(value_type = String)]
Reqwest(Arc<reqwest::Error>),
/// Server returned a status code different from 200.
#[display("Server returned a status code {status_code}.")]
BadStatusCode { status_code: u16 },
/// The url leads to a file of a different type.
#[display("The link leads to a file of type '{content_type}'.")]
BadContentType { content_type: String },
/// Server doesn't return expected headers.
#[display("Server doesn't return expected header(s) '{expected_header}'.")]
BadHeaders { expected_header: String },
}
impl FetchError {
pub fn unknown(error: Arc<reqwest::Error>) -> Self {
Self::Reqwest(error)
}
pub fn bad_status_code(status_code: u16) -> Self {
Self::BadStatusCode { status_code }
}
pub fn bad_content_type(content_type: &str) -> Self {
Self::BadContentType {
content_type: content_type.to_string(),
}
}
pub fn bad_headers(expected_header: &str) -> Self {
Self::BadHeaders {
expected_header: expected_header.to_string(),
}
}
}
impl PartialEq for FetchError {
fn eq(&self, other: &Self) -> bool {
discriminant(self) == discriminant(other)
}
}
/// Result of XLS data retrieval.
#[derive(Debug, PartialEq)]
pub struct FetchOk {
/// File upload date.
pub uploaded_at: DateTime<Utc>,
/// Date data received.
pub requested_at: DateTime<Utc>,
/// Etag.
pub etag: String,
/// File data.
pub data: Option<Vec<u8>>,
}
impl FetchOk {
/// Result without file content.
pub fn head(uploaded_at: DateTime<Utc>, etag: String) -> Self {
FetchOk {
uploaded_at,
requested_at: Utc::now(),
etag,
data: None,
}
}
/// Full result.
pub fn get(uploaded_at: DateTime<Utc>, etag: String, data: Vec<u8>) -> Self {
FetchOk {
uploaded_at,
requested_at: Utc::now(),
etag,
data: Some(data),
}
}
}
pub type FetchResult = Result<FetchOk, FetchError>;
pub struct XlsDownloader {
pub url: Option<String>,
pub etag: Option<String>,
}
impl XlsDownloader {
pub fn new() -> Self {
XlsDownloader {
url: None,
etag: None,
}
}
async fn fetch_specified(url: &str, head: bool) -> FetchResult {
let client = reqwest::Client::new();
let response = if head {
client.head(url)
} else {
client.get(url)
}
.header("User-Agent", ua_generator::ua::spoof_chrome_ua())
.send()
.await
.map_err(|e| FetchError::unknown(Arc::new(e)))?;
if response.status().as_u16() != 200 {
return Err(FetchError::bad_status_code(response.status().as_u16()));
}
let headers = response.headers();
let content_type = headers
.get("Content-Type")
.ok_or(FetchError::bad_headers("Content-Type"))?;
let etag = headers
.get("etag")
.ok_or(FetchError::bad_headers("etag"))?
.to_str()
.or(Err(FetchError::bad_headers("etag")))?
.to_string();
let last_modified = headers
.get("last-modified")
.ok_or(FetchError::bad_headers("last-modified"))?;
if content_type != "application/vnd.ms-excel" {
return Err(FetchError::bad_content_type(content_type.to_str().unwrap()));
}
let last_modified = DateTime::parse_from_rfc2822(last_modified.to_str().unwrap())
.unwrap()
.with_timezone(&Utc);
Ok(if head {
FetchOk::head(last_modified, etag)
} else {
FetchOk::get(
last_modified,
etag,
response.bytes().await.unwrap().to_vec(),
)
})
}
pub async fn fetch(&self, head: bool) -> FetchResult {
if self.url.is_none() {
Err(FetchError::NoUrlProvided)
} else {
Self::fetch_specified(self.url.as_ref().unwrap(), head).await
}
}
pub async fn set_url(&mut self, url: &str) -> FetchResult {
let result = Self::fetch_specified(url, true).await;
if result.is_ok() {
self.url = Some(url.to_string());
}
result
}
}
#[cfg(test)]
mod tests {
use crate::xls_downloader::{FetchError, XlsDownloader};
#[tokio::test]
async fn bad_url() {
let url = "bad_url";
let mut downloader = XlsDownloader::new();
assert!(downloader.set_url(url).await.is_err());
}
#[tokio::test]
async fn bad_status_code() {
let url = "https://www.google.com/not-found";
let mut downloader = XlsDownloader::new();
assert_eq!(
downloader.set_url(url).await,
Err(FetchError::bad_status_code(404))
);
}
#[tokio::test]
async fn bad_headers() {
let url = "https://www.google.com/favicon.ico";
let mut downloader = XlsDownloader::new();
assert_eq!(
downloader.set_url(url).await,
Err(FetchError::BadHeaders {
expected_header: "ETag".to_string(),
})
);
}
#[tokio::test]
async fn bad_content_type() {
let url = "https://s3.aero-storage.ldragol.ru/679e5d1145a6ad00843ad3f1/67ddb59fd46303008396ac96%2Fexample.txt";
let mut downloader = XlsDownloader::new();
assert!(downloader.set_url(url).await.is_err());
}
#[tokio::test]
async fn ok() {
let url = "https://s3.aero-storage.ldragol.ru/679e5d1145a6ad00843ad3f1/67ddb5fad46303008396ac97%2Fschedule.xls";
let mut downloader = XlsDownloader::new();
assert!(downloader.set_url(url).await.is_ok());
}
#[tokio::test]
async fn downloader_ok() {
let url = "https://s3.aero-storage.ldragol.ru/679e5d1145a6ad00843ad3f1/67ddb5fad46303008396ac97%2Fschedule.xls";
let mut downloader = XlsDownloader::new();
assert!(downloader.set_url(url).await.is_ok());
assert!(downloader.fetch(false).await.is_ok());
}
#[tokio::test]
async fn downloader_no_url_provided() {
let downloader = XlsDownloader::new();
let result = downloader.fetch(false).await;
assert_eq!(result, Err(FetchError::NoUrlProvided));
}
}
@@ -0,0 +1,99 @@
use chrono::{DateTime, Utc};
use derive_more::{Display, Error};
use std::mem::discriminant;
use std::sync::Arc;
mod yandex_disk;
/// XLS data retrieval errors.
#[derive(Clone, Debug, Display, Error)]
pub enum FetchError {
/// Unknown error.
#[display("An unknown error occurred while downloading the file.")]
Reqwest(Arc<reqwest::Error>),
/// Server returned a status code different from 200.
#[display("Server returned a status code {status_code}.")]
BadStatusCode { status_code: u16 },
/// The folder contains no file matching the schedule name pattern.
#[display("No schedule file was found in the shared folder.")]
NoScheduleFile,
}
impl FetchError {
pub fn unknown(error: Arc<reqwest::Error>) -> Self {
Self::Reqwest(error)
}
pub fn bad_status_code(status_code: u16) -> Self {
Self::BadStatusCode { status_code }
}
}
impl PartialEq for FetchError {
fn eq(&self, other: &Self) -> bool {
discriminant(self) == discriminant(other)
}
}
pub type FetchResult<T> = Result<T, FetchError>;
/// Description of the remote schedule file, obtained without downloading its content.
#[derive(Clone, Debug, PartialEq)]
pub struct RemoteFile {
/// Permanent link to the file, shown to API clients.
pub url: String,
/// Link the content is actually downloaded from.
pub download_url: String,
/// Content hash, changing whenever the file content changes.
pub version: String,
/// Time of the last file modification reported by the remote side.
pub modified_at: DateTime<Utc>,
}
/// Public Yandex Disk folder the schedule is downloaded from.
#[derive(Clone, Debug)]
pub struct Source {
pub public_url: String,
}
impl Source {
pub fn new(public_url: String) -> Self {
Self { public_url }
}
/// Looks up the current schedule file without downloading its content.
pub async fn probe(&self) -> FetchResult<RemoteFile> {
yandex_disk::probe(&self.public_url).await
}
/// Downloads the content of a previously probed file.
pub async fn download(&self, file: &RemoteFile) -> FetchResult<Vec<u8>> {
get(&file.download_url)
.await?
.bytes()
.await
.map(|bytes| bytes.to_vec())
.map_err(|error| FetchError::unknown(Arc::new(error)))
}
}
/// Performs a GET request with a spoofed browser User-Agent and checks the status code.
async fn get(url: &str) -> FetchResult<reqwest::Response> {
let response = reqwest::Client::new()
.get(url)
.header("User-Agent", ua_generator::ua::spoof_chrome_ua())
.send()
.await
.map_err(|error| FetchError::unknown(Arc::new(error)))?;
if response.status().as_u16() != 200 {
return Err(FetchError::bad_status_code(response.status().as_u16()));
}
Ok(response)
}
@@ -0,0 +1,135 @@
use super::{FetchError, FetchResult, RemoteFile};
use chrono::{DateTime, Utc};
use percent_encoding::{AsciiSet, CONTROLS, NON_ALPHANUMERIC, utf8_percent_encode};
use serde::Deserialize;
/// Prefix of the schedule file names in the shared folder.
const NAME_PREFIX: &str = "poltavskaja_";
/// Marker of the corrections file, which holds a separate schedule.
const NAME_EXCLUDED_MARKER: &str = "korr";
/// Extension of the schedule files.
const NAME_SUFFIX: &str = ".xls";
/// Maximum amount of entries requested from the folder listing.
const LISTING_LIMIT: u32 = 200;
/// Characters not allowed inside a single path segment of the public file link.
const PATH_SEGMENT: &AsciiSet = &CONTROLS
.add(b' ')
.add(b'"')
.add(b'#')
.add(b'%')
.add(b'/')
.add(b'<')
.add(b'>')
.add(b'?')
.add(b'`')
.add(b'{')
.add(b'}');
#[derive(Deserialize)]
struct Listing {
#[serde(rename = "_embedded")]
embedded: Embedded,
}
#[derive(Deserialize)]
struct Embedded {
items: Vec<Item>,
}
#[derive(Deserialize)]
struct Item {
#[serde(rename = "type")]
resource_type: String,
name: String,
modified: DateTime<Utc>,
md5: Option<String>,
revision: Option<u64>,
file: Option<String>,
}
impl Item {
/// Whether the entry is the schedule the provider is interested in.
fn is_schedule(&self) -> bool {
if self.resource_type != "file" || self.file.is_none() {
return false;
}
let name = self.name.to_lowercase();
name.starts_with(NAME_PREFIX)
&& name.ends_with(NAME_SUFFIX)
&& !name.contains(NAME_EXCLUDED_MARKER)
}
/// Marker changing whenever the file content changes.
fn version(&self) -> String {
self.md5
.clone()
.or_else(|| self.revision.map(|revision| revision.to_string()))
.unwrap_or_else(|| self.modified.to_rfc3339())
}
}
/// Finds the freshest schedule file in the public folder.
///
/// The files inside the folder are replaced independently of the folder link,
/// so the whole listing is re-read on every probe.
pub async fn probe(public_url: &str) -> FetchResult<RemoteFile> {
let listing = super::get(&format!(
"https://cloud-api.yandex.net/v1/disk/public/resources?public_key={}&limit={}",
utf8_percent_encode(public_url, NON_ALPHANUMERIC),
LISTING_LIMIT
))
.await?
.json::<Listing>()
.await
.map_err(|error| FetchError::unknown(std::sync::Arc::new(error)))?;
let item = listing
.embedded
.items
.into_iter()
.filter(Item::is_schedule)
.max_by_key(|item| (item.modified, item.revision))
.ok_or(FetchError::NoScheduleFile)?;
Ok(RemoteFile {
url: format!(
"{}/{}",
public_url.trim_end_matches('/'),
utf8_percent_encode(&item.name, PATH_SEGMENT)
),
version: item.version(),
modified_at: item.modified,
download_url: item.file.unwrap(),
})
}
#[cfg(test)]
mod tests {
use super::probe;
const PUBLIC_URL: &str = "https://disk.yandex.ru/d/e8HJpMgDq7msyg";
#[tokio::test]
async fn probe_ok() {
let file = probe(PUBLIC_URL).await.unwrap();
assert!(file.url.starts_with(PUBLIC_URL));
assert!(!file.version.is_empty());
assert!(file.download_url.starts_with("https://"));
}
#[tokio::test]
async fn probe_unknown_folder() {
assert!(
probe("https://disk.yandex.ru/d/000000000000000")
.await
.is_err()
);
}
}
+5 -7
View File
@@ -1,14 +1,14 @@
use crate::middlewares::authorization::{JWTAuthorizationBuilder, ServiceConfig}; use crate::middlewares::authorization::{JWTAuthorizationBuilder, ServiceConfig};
use crate::middlewares::content_type::ContentTypeBootstrap; use crate::middlewares::content_type::ContentTypeBootstrap;
use crate::state::{new_app_state, AppState}; use crate::state::{AppState, new_app_state};
use actix_web::dev::{ServiceFactory, ServiceRequest}; use actix_web::dev::{ServiceFactory, ServiceRequest};
use actix_web::{App, Error, HttpServer}; use actix_web::{App, Error, HttpServer};
use database::entity::sea_orm_active_enums::UserRole; use database::entity::sea_orm_active_enums::UserRole;
use dotenvy::dotenv; use dotenvy::dotenv;
use log::info; use log::info;
use std::io; use std::io;
use utoipa_actix_web::scope::Scope;
use utoipa_actix_web::AppExt; use utoipa_actix_web::AppExt;
use utoipa_actix_web::scope::Scope;
use utoipa_rapidoc::RapiDoc; use utoipa_rapidoc::RapiDoc;
mod state; mod state;
@@ -158,11 +158,9 @@ async fn async_main() -> io::Result<()> {
fn main() -> io::Result<()> { fn main() -> io::Result<()> {
let _guard = sentry::init(( let _guard = sentry::init((
"https://9c33db76e89984b3f009b28a9f4b5954@sentry.n08i40k.ru/8", "https://9c33db76e89984b3f009b28a9f4b5954@sentry.n08i40k.ru/8",
sentry::ClientOptions { sentry::ClientOptions::new()
release: sentry::release_name!(), .maybe_release(sentry::release_name!())
send_default_pii: true, .send_default_pii(true),
..Default::default()
},
)); ));
let _ = dotenv(); let _ = dotenv();
+8 -3
View File
@@ -146,8 +146,8 @@ mod tests {
use actix_web::http::StatusCode; use actix_web::http::StatusCode;
use actix_web::test; use actix_web::test;
use database::entity::sea_orm_active_enums::UserRole; use database::entity::sea_orm_active_enums::UserRole;
use database::entity::ActiveUser; use database::entity::{ActiveUser, UserEntity};
use database::sea_orm::{ActiveModelTrait, Set}; use database::sea_orm::{ActiveModelTrait, EntityTrait, Set};
use sha1::{Digest, Sha1}; use sha1::{Digest, Sha1};
use std::fmt::Write; use std::fmt::Write;
@@ -193,8 +193,13 @@ mod tests {
android_version: Set(None), android_version: Set(None),
}; };
UserEntity::delete_by_id(&id)
.exec(app_state.get_database())
.await
.expect("Failed to delete user");
active_user active_user
.save(app_state.get_database()) .insert(app_state.get_database())
.await .await
.expect("Failed to save user"); .expect("Failed to save user");
} }
-9
View File
@@ -2,22 +2,13 @@ pub mod schedule;
pub mod telegram; pub mod telegram;
pub mod vk_id; pub mod vk_id;
#[cfg(not(test))]
pub mod yandex_cloud;
pub use self::schedule::ScheduleEnvData; pub use self::schedule::ScheduleEnvData;
pub use self::telegram::TelegramEnvData; pub use self::telegram::TelegramEnvData;
pub use self::vk_id::VkIdEnvData; pub use self::vk_id::VkIdEnvData;
#[cfg(not(test))]
pub use self::yandex_cloud::YandexCloudEnvData;
#[derive(Default)] #[derive(Default)]
pub struct AppEnv { pub struct AppEnv {
pub schedule: ScheduleEnvData, pub schedule: ScheduleEnvData,
pub telegram: TelegramEnvData, pub telegram: TelegramEnvData,
pub vk_id: VkIdEnvData, pub vk_id: VkIdEnvData,
#[cfg(not(test))]
pub yandex_cloud: YandexCloudEnvData,
} }
+5 -2
View File
@@ -2,8 +2,10 @@ use std::env;
#[derive(Clone)] #[derive(Clone)]
pub struct ScheduleEnvData { pub struct ScheduleEnvData {
/// Public link to the Yandex Disk folder the schedule files are uploaded to.
#[cfg(not(test))] #[cfg(not(test))]
pub url: Option<String>, pub yandex_disk_url: String,
pub auto_update: bool, pub auto_update: bool,
} }
@@ -11,7 +13,8 @@ impl Default for ScheduleEnvData {
fn default() -> Self { fn default() -> Self {
Self { Self {
#[cfg(not(test))] #[cfg(not(test))]
url: env::var("SCHEDULE_INIT_URL").ok(), yandex_disk_url: env::var("SCHEDULE_YANDEX_DISK_URL")
.expect("SCHEDULE_YANDEX_DISK_URL must be set"),
auto_update: !env::var("SCHEDULE_DISABLE_AUTO_UPDATE") auto_update: !env::var("SCHEDULE_DISABLE_AUTO_UPDATE")
.is_ok_and(|v| v.eq("1") || v.eq("true")), .is_ok_and(|v| v.eq("1") || v.eq("true")),
} }
-16
View File
@@ -1,16 +0,0 @@
use std::env;
#[derive(Clone)]
pub struct YandexCloudEnvData {
pub api_key: String,
pub func_id: String,
}
impl Default for YandexCloudEnvData {
fn default() -> Self {
Self {
api_key: env::var("YANDEX_CLOUD_API_KEY").expect("YANDEX_CLOUD_API_KEY must be set"),
func_id: env::var("YANDEX_CLOUD_FUNC_ID").expect("YANDEX_CLOUD_FUNC_ID must be set"),
}
}
}
+41 -16
View File
@@ -38,13 +38,8 @@ impl AppState {
#[cfg(not(test))] #[cfg(not(test))]
{ {
if let Some(url) = &env.schedule.url { providers::EngelsPolytechnicUpdateSource::YandexDisk {
providers::EngelsPolytechnicUpdateSource::Url(url.clone()) public_url: env.schedule.yandex_disk_url.clone(),
} else {
providers::EngelsPolytechnicUpdateSource::GrabFromSite {
yandex_api_key: env.yandex_cloud.api_key.clone(),
yandex_func_id: env.yandex_cloud.func_id.clone(),
}
} }
} }
}) })
@@ -56,15 +51,8 @@ impl AppState {
database: if let Some(database) = database { database: if let Some(database) = database {
database database
} else { } else {
let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be set"); let opt = database_connect_options();
let database_url = opt.get_url().to_string();
let mut opt = ConnectOptions::new(database_url.clone());
opt.max_connections(4)
.min_connections(2)
.connect_timeout(Duration::from_secs(10))
.idle_timeout(Duration::from_secs(8))
.sqlx_logging(true);
let database = Database::connect(opt) let database = Database::connect(opt)
.await .await
@@ -109,6 +97,43 @@ impl AppState {
} }
} }
/// Параметры подключения к базе данных.
#[cfg(not(test))]
fn database_connect_options() -> ConnectOptions {
let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be set");
let mut opt = ConnectOptions::new(database_url);
opt.max_connections(4)
.min_connections(2)
.connect_timeout(Duration::from_secs(10))
.idle_timeout(Duration::from_secs(8))
.sqlx_logging(true);
opt
}
/// SQLite во временном файле: база в памяти умрёт вместе с соединением.
#[cfg(test)]
fn database_connect_options() -> ConnectOptions {
let path = std::env::temp_dir().join(format!(
"{}-test-{}.sqlite",
env!("CARGO_PKG_NAME"),
std::process::id()
));
let _ = std::fs::remove_file(&path);
let mut opt = ConnectOptions::new(format!("sqlite://{}?mode=rwc", path.display()));
opt.max_connections(1)
.min_connections(1)
.connect_timeout(Duration::from_secs(10))
.sqlx_logging(true);
opt
}
/// Create a new object web::Data<AppState>. /// Create a new object web::Data<AppState>.
pub async fn new_app_state( pub async fn new_app_state(
database: Option<DatabaseConnection>, database: Option<DatabaseConnection>,
+14 -8
View File
@@ -1,10 +1,13 @@
use aws_lc_rs::signature::{ED25519, UnparsedPublicKey};
use base64::Engine; use base64::Engine;
use derive_more::{Display, Error}; use derive_more::{Display, Error};
use ed25519_dalek::Verifier;
use hex_literal::hex; use hex_literal::hex;
use serde::Deserialize; use serde::Deserialize;
use std::collections::HashMap; use std::collections::HashMap;
/// Длина подписи Ed25519 в байтах.
const ED25519_SIGNATURE_LENGTH: usize = 64;
pub struct WebAppInitDataMap { pub struct WebAppInitDataMap {
pub data_map: HashMap<String, String>, pub data_map: HashMap<String, String>,
} }
@@ -54,10 +57,10 @@ impl WebAppInitDataMap {
hex!("40055058a4ee38156a06562e52eece92a771bcd8346a8c4615cb7376eddf72ec"), hex!("40055058a4ee38156a06562e52eece92a771bcd8346a8c4615cb7376eddf72ec"),
]; ];
let verifying_key = ed25519_dalek::VerifyingKey::from_bytes( let verifying_key = UnparsedPublicKey::new(
&TELEGRAM_PUBLIC_KEY[if test_dc { 1 } else { 0 }], &ED25519,
) TELEGRAM_PUBLIC_KEY[if test_dc { 1 } else { 0 }],
.unwrap(); );
let signature = { let signature = {
let raw = self let raw = self
@@ -69,8 +72,11 @@ impl WebAppInitDataMap {
.decode(raw) .decode(raw)
.map_err(|_| VerifyError::BadSignature)?; .map_err(|_| VerifyError::BadSignature)?;
ed25519_dalek::Signature::from_slice(bytes.as_slice()) if bytes.len() != ED25519_SIGNATURE_LENGTH {
.map_err(|_| VerifyError::BadSignature)? return Err(VerifyError::BadSignature);
}
bytes
}; };
let data_check_string = format!("{}:WebAppData\n{}", bot_id, { let data_check_string = format!("{}:WebAppData\n{}", bot_id, {
@@ -85,7 +91,7 @@ impl WebAppInitDataMap {
}); });
verifying_key verifying_key
.verify(data_check_string.as_bytes(), &signature) .verify(data_check_string.as_bytes(), signature.as_slice())
.map_err(|_| VerifyError::IntegrityCheckFailed) .map_err(|_| VerifyError::IntegrityCheckFailed)
} }
} }