From a6c84302b236e01205d72a47e9cb625db39c1b60 Mon Sep 17 00:00:00 2001 From: n08i40k Date: Mon, 16 Sep 2024 15:12:29 +0400 Subject: [PATCH] =?UTF-8?q?=D0=9E=D0=B1=D1=85=D0=BE=D0=B4=20=D0=B1=D0=B0?= =?UTF-8?q?=D0=BD=D0=B0=20=D0=BF=D0=BE=20IP=20=D0=B7=D0=B0=20=D1=87=D0=B0?= =?UTF-8?q?=D1=81=D1=82=D1=8B=D0=B5=20=D0=BE=D0=B1=D1=80=D0=B0=D1=89=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D1=8F=20=D0=BA=20=D1=81=D0=B0=D0=B9=D1=82=D1=83=20?= =?UTF-8?q?=D0=BF=D0=BE=D0=BB=D0=B8=D1=82=D0=B5=D1=85=D0=BD=D0=B8=D0=BA?= =?UTF-8?q?=D1=83=D0=BC=D0=B0=20=D0=B7=D0=B0=20=D1=81=D1=87=D1=91=D1=82=20?= =?UTF-8?q?=D1=81=D0=BA=D0=B0=D1=87=D0=B8=D0=B2=D0=B0=D0=BD=D0=B8=D1=8F=20?= =?UTF-8?q?=D0=BA=D0=BE=D0=B4=D0=B0=20=D1=81=D1=82=D1=80=D0=B0=D0=BD=D0=B8?= =?UTF-8?q?=D1=86=D1=8B=20=D0=BD=D0=B0=20=D1=81=D1=82=D0=BE=D1=80=D0=BE?= =?UTF-8?q?=D0=BD=D0=B5=20=D0=BA=D0=BB=D0=B8=D0=B5=D0=BD=D1=82=D0=BE=D0=B2?= =?UTF-8?q?.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/dto/schedule.dto.ts | 27 ++++++ .../schedule-parser/schedule-parser.ts | 15 ++++ .../xls-downloader/basic-xls-downloader.ts | 90 ++++++++++++------- .../xls-downloader/xls-downloader.base.ts | 5 ++ src/schedule/schedule.controller.ts | 30 +++++-- src/schedule/schedule.service.ts | 23 ++++- 6 files changed, 153 insertions(+), 37 deletions(-) diff --git a/src/dto/schedule.dto.ts b/src/dto/schedule.dto.ts index 69fbf95..5bc50a7 100644 --- a/src/dto/schedule.dto.ts +++ b/src/dto/schedule.dto.ts @@ -1,5 +1,7 @@ import { IsArray, + IsBase64, + IsBoolean, IsDate, IsEnum, IsNumber, @@ -240,6 +242,14 @@ export class ScheduleDto { }) @Type(() => Object) lastChangedDays: Array>; + + @ApiProperty({ + example: false, + description: + "Требуется ли пользовательское обновление ссылки для скачивания расписания", + }) + @IsBoolean() + updateRequired: boolean; } export class GroupScheduleRequestDto extends PickType(GroupDto, ["name"]) {} @@ -269,4 +279,21 @@ export class GroupScheduleDto extends OmitType(ScheduleDto, [ @ValidateNested({ each: true }) @Type(() => Number) lastChangedDays: Array; + + @ApiProperty({ + example: false, + description: + "Требуется ли пользовательское обновление ссылки для скачивания расписания", + }) + @IsBoolean() + updateRequired: boolean; +} + +export class SiteMainPageDto { + @ApiProperty({ + example: "
", + description: "Код страницы политехникума для скачивания", + }) + @IsBase64() + mainPage: string; } diff --git a/src/schedule/internal/schedule-parser/schedule-parser.ts b/src/schedule/internal/schedule-parser/schedule-parser.ts index e0eca7a..f9c1453 100644 --- a/src/schedule/internal/schedule-parser/schedule-parser.ts +++ b/src/schedule/internal/schedule-parser/schedule-parser.ts @@ -20,6 +20,7 @@ export class ScheduleParseResult { etag: string; groups: Array; affectedDays: Array>; + updateRequired: boolean; } export class ScheduleParser { @@ -114,6 +115,10 @@ export class ScheduleParser { return { daySkeletons: days, groupSkeletons: groups }; } + getXlsDownloader(): XlsDownloaderBase { + return this.xlsDownloader; + } + async getSchedule( forceCached: boolean = false, ): Promise { @@ -121,6 +126,15 @@ export class ScheduleParser { const downloadData = await this.xlsDownloader.downloadXLS(); + if (downloadData.updateRequired && downloadData.etag.length === 0) { + return { + updateRequired: true, + groups: [], + etag: "", + affectedDays: [], + }; + } + if ( !downloadData.new && this.lastResult && @@ -241,6 +255,7 @@ export class ScheduleParser { etag: downloadData.etag, groups: groups, affectedDays: this.getAffectedDays(this.lastResult?.groups, groups), + updateRequired: downloadData.updateRequired, }); } diff --git a/src/schedule/internal/xls-downloader/basic-xls-downloader.ts b/src/schedule/internal/xls-downloader/basic-xls-downloader.ts index 9b90f6d..f910c7e 100644 --- a/src/schedule/internal/xls-downloader/basic-xls-downloader.ts +++ b/src/schedule/internal/xls-downloader/basic-xls-downloader.ts @@ -5,47 +5,57 @@ import { } from "./xls-downloader.base"; import axios from "axios"; import { JSDOM } from "jsdom"; +import { NotAcceptableException } from "@nestjs/common"; export class BasicXlsDownloader extends XlsDownloaderBase { cache: XlsDownloaderResult | null = null; + preparedData: { downloadLink: string; updateDate: string } | null = null; + private lastUpdate: number = 0; - private async getDOM(): Promise { - const response = await axios.get(this.url); - - if (response.status !== 200) { - throw new Error(`Не удалось получить данные с основной страницы! -Статус код: ${response.status} -${response.statusText}`); + private async getDOM(preparedData: any): Promise { + try { + return new JSDOM(atob(preparedData), { + url: this.url, + contentType: "text/html", + }); + } catch { + throw new NotAcceptableException( + "Передан некорректный код страницы", + ); } - - return new JSDOM(response.data, { - url: this.url, - contentType: "text/html", - }); } private parseData(dom: JSDOM): { downloadLink: string; updateDate: string; } { - const scheduleBlock = dom.window.document.getElementById("cont-i"); - if (scheduleBlock === null) - throw new Error("Не удалось найти блок расписаний!"); + try { + const scheduleBlock = dom.window.document.getElementById("cont-i"); + if (scheduleBlock === null) + // noinspection ExceptionCaughtLocallyJS + throw new Error("Не удалось найти блок расписаний!"); - const schedules = scheduleBlock.getElementsByTagName("div"); - if (schedules === null || schedules.length === 0) - throw new Error("Не удалось найти строку с расписанием!"); + const schedules = scheduleBlock.getElementsByTagName("div"); + if (schedules === null || schedules.length === 0) + // noinspection ExceptionCaughtLocallyJS + throw new Error("Не удалось найти строку с расписанием!"); - const poltavskaya = schedules[0]; - const link = poltavskaya.getElementsByTagName("a")[0]!; + const poltavskaya = schedules[0]; + const link = poltavskaya.getElementsByTagName("a")[0]!; - const spans = poltavskaya.getElementsByTagName("span"); - const updateDate = spans[3].textContent!.trimStart(); + const spans = poltavskaya.getElementsByTagName("span"); + const updateDate = spans[3].textContent!.trimStart(); - return { - downloadLink: link.href, - updateDate: updateDate, - }; + return { + downloadLink: link.href, + updateDate: updateDate, + }; + } catch (exception) { + console.error(exception); + throw new NotAcceptableException( + "Передан некорректный код страницы", + ); + } } public async getCachedXLS(): Promise { @@ -56,6 +66,17 @@ ${response.statusText}`); return this.cache; } + public isUpdateRequired(): boolean { + return (Date.now() - this.lastUpdate) / 1000 / 60 > 5; + } + + public async setPreparedData(preparedData: string): Promise { + const dom = await this.getDOM(preparedData); + this.preparedData = this.parseData(dom); + + this.lastUpdate = Date.now(); + } + public async downloadXLS(): Promise { if ( this.cacheMode === XlsDownloaderCacheMode.HARD && @@ -63,12 +84,18 @@ ${response.statusText}`); ) return this.getCachedXLS(); - const dom = await this.getDOM(); - const parseData = this.parseData(dom); + if (!this.preparedData) { + return { + updateRequired: true, + etag: "", + new: true, + fileData: new ArrayBuffer(1), + updateDate: "", + }; + } - // FIX-ME: Что такое Annotator и почему он выдаёт пустое предупреждение? // noinspection Annotator - const response = await axios.get(parseData.downloadLink, { + const response = await axios.get(this.preparedData.downloadLink, { responseType: "arraybuffer", }); if (response.status !== 200) { @@ -79,12 +106,13 @@ ${response.statusText}`); const result: XlsDownloaderResult = { fileData: response.data.buffer, - updateDate: parseData.updateDate, + updateDate: this.preparedData.updateDate, etag: response.headers["etag"], new: this.cacheMode === XlsDownloaderCacheMode.NONE ? true : this.cache?.etag !== response.headers["etag"], + updateRequired: this.isUpdateRequired(), }; if (this.cacheMode !== XlsDownloaderCacheMode.NONE) this.cache = result; diff --git a/src/schedule/internal/xls-downloader/xls-downloader.base.ts b/src/schedule/internal/xls-downloader/xls-downloader.base.ts index 21060a8..df90fe3 100644 --- a/src/schedule/internal/xls-downloader/xls-downloader.base.ts +++ b/src/schedule/internal/xls-downloader/xls-downloader.base.ts @@ -3,6 +3,7 @@ export type XlsDownloaderResult = { updateDate: string; etag: string; new: boolean; + updateRequired: boolean; }; export enum XlsDownloaderCacheMode { @@ -21,6 +22,10 @@ export abstract class XlsDownloaderBase { public abstract getCachedXLS(): Promise; + public abstract isUpdateRequired(): boolean; + + public abstract setPreparedData(preparedData: string): Promise; + public getCacheMode(): XlsDownloaderCacheMode { return this.cacheMode; } diff --git a/src/schedule/schedule.controller.ts b/src/schedule/schedule.controller.ts index ad8146c..5d3a1bd 100644 --- a/src/schedule/schedule.controller.ts +++ b/src/schedule/schedule.controller.ts @@ -14,10 +14,12 @@ import { GroupScheduleRequestDto, ScheduleDto, ScheduleGroupsDto, + SiteMainPageDto, } from "../dto/schedule.dto"; import { ResultDto } from "../utility/validation/class-validator.interceptor"; import { ApiExtraModels, + ApiNotAcceptableResponse, ApiNotFoundResponse, ApiOkResponse, ApiOperation, @@ -38,8 +40,8 @@ export class ScheduleController { @ResultDto(ScheduleDto) @HttpCode(HttpStatus.OK) @Get("get") - getSchedule(): Promise { - return this.scheduleService.getSchedule(); + async getSchedule(): Promise { + return await this.scheduleService.getSchedule(); } @ApiExtraModels(GroupScheduleDto) @@ -55,10 +57,10 @@ export class ScheduleController { @ResultDto(GroupScheduleDto) @HttpCode(HttpStatus.OK) @Post("get-group") - getGroupSchedule( + async getGroupSchedule( @Body() groupDto: GroupScheduleRequestDto, ): Promise { - return this.scheduleService.getGroup(groupDto.name); + return await this.scheduleService.getGroup(groupDto.name); } @ApiExtraModels(ScheduleGroupsDto) @@ -75,6 +77,24 @@ export class ScheduleController { @HttpCode(HttpStatus.OK) @Get("get-group-names") async getGroupNames(): Promise { - return this.scheduleService.getGroupNames(); + return await this.scheduleService.getGroupNames(); + } + + @ApiExtraModels(SiteMainPageDto) + @ApiOperation({ + summary: "Обновление данных основной страницы политехникума", + tags: ["schedule"], + }) + @ApiOkResponse({ description: "Данные обновлены успешно" }) + @ApiNotAcceptableResponse({ + description: "Передан некорректный код страницы", + }) + @ResultDto(null) + @HttpCode(HttpStatus.OK) + @Post("update-site-main-page") + async updateSiteMainPage( + @Body() siteMainPageDto: SiteMainPageDto, + ): Promise { + return await this.scheduleService.updateSiteMainPage(siteMainPageDto); } } diff --git a/src/schedule/schedule.service.ts b/src/schedule/schedule.service.ts index f566d38..58bbb99 100644 --- a/src/schedule/schedule.service.ts +++ b/src/schedule/schedule.service.ts @@ -1,4 +1,9 @@ -import { Inject, Injectable, NotFoundException } from "@nestjs/common"; +import { + Inject, + Injectable, + NotFoundException, + ServiceUnavailableException, +} from "@nestjs/common"; import { ScheduleParser, ScheduleParseResult, @@ -10,6 +15,7 @@ import { GroupScheduleDto, ScheduleDto, ScheduleGroupsDto, + SiteMainPageDto, } from "../dto/schedule.dto"; import { Cache, CACHE_MANAGER } from "@nestjs/cache-manager"; import { instanceToPlain } from "class-transformer"; @@ -38,6 +44,11 @@ export class ScheduleService { schedule.groups, ) as Array; + if (schedule.updateRequired && schedule.etag.length === 0) + throw new ServiceUnavailableException( + "Отсутствует начальная ссылка на скачивание!", + ); + return schedule; }); } @@ -66,6 +77,7 @@ export class ScheduleService { groups: ScheduleService.toObject(sourceSchedule.groups), etag: sourceSchedule.etag, lastChangedDays: this.lastChangedDays, + updateRequired: sourceSchedule.updateRequired, }; }); } @@ -84,6 +96,7 @@ export class ScheduleService { group: schedule.groups[group], etag: schedule.etag, lastChangedDays: this.lastChangedDays[group] ?? [], + updateRequired: schedule.updateRequired, }; } @@ -107,4 +120,12 @@ export class ScheduleService { return groupNames; } + + async updateSiteMainPage(siteMainPageDto: SiteMainPageDto): Promise { + await this.scheduleParser + .getXlsDownloader() + .setPreparedData(siteMainPageDto.mainPage); + + await this.cacheManager.reset(); + } }