From cd9dc319ebe578dc03637f49efcd004170f93308 Mon Sep 17 00:00:00 2001 From: n08i40k Date: Thu, 26 Sep 2024 01:37:44 +0400 Subject: [PATCH] =?UTF-8?q?=D0=9E=D0=B1=D0=BD=D0=BE=D0=B2=D0=BB=D1=91?= =?UTF-8?q?=D0=BD=D0=BD=D0=B0=D1=8F=20=D1=81=D0=B8=D1=81=D1=82=D0=B5=D0=BC?= =?UTF-8?q?=D0=B0=20=D0=BA=D0=B5=D1=88=D0=B8=D1=80=D0=BE=D0=B2=D0=B0=D0=BD?= =?UTF-8?q?=D0=B8=D1=8F=20=D0=B8=20=D1=87=D0=B8=D1=81=D1=82=D0=BA=D0=B0=20?= =?UTF-8?q?=D0=BA=D0=BE=D0=B4=D0=B0.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 91 +------------------ package.json | 2 +- src/dto/schedule.dto.ts | 44 ++++----- .../schedule-parser/schedule-parser.ts | 25 +++-- .../xls-downloader/basic-xls-downloader.ts | 15 ++- src/schedule/schedule.controller.ts | 14 +++ src/schedule/schedule.service.ts | 69 +++++++------- 7 files changed, 95 insertions(+), 165 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2a9bde8..327a9e1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,7 +28,7 @@ "rxjs": "^7.8.1", "swagger-ui-express": "^5.0.1", "uuid": "^10.0.0", - "xlsx": "^0.18.5" + "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz" }, "devDependencies": { "@nestjs/cli": "^10.0.0", @@ -2798,14 +2798,6 @@ "node": ">=0.4.0" } }, - "node_modules/adler-32": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz", - "integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==", - "engines": { - "node": ">=0.8" - } - }, "node_modules/agent-base": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", @@ -3458,18 +3450,6 @@ } ] }, - "node_modules/cfb": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz", - "integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==", - "dependencies": { - "adler-32": "~1.3.0", - "crc-32": "~1.2.0" - }, - "engines": { - "node": ">=0.8" - } - }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -3675,14 +3655,6 @@ "node": ">= 0.12.0" } }, - "node_modules/codepage": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz", - "integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==", - "engines": { - "node": ">=0.8" - } - }, "node_modules/collect-v8-coverage": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.2.tgz", @@ -3874,17 +3846,6 @@ } } }, - "node_modules/crc-32": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", - "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", - "bin": { - "crc32": "bin/crc32.njs" - }, - "engines": { - "node": ">=0.8" - } - }, "node_modules/create-jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz", @@ -5030,14 +4991,6 @@ "node": ">= 0.6" } }, - "node_modules/frac": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz", - "integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==", - "engines": { - "node": ">=0.8" - } - }, "node_modules/fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", @@ -8550,17 +8503,6 @@ "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dev": true }, - "node_modules/ssf": { - "version": "0.11.2", - "resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz", - "integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==", - "dependencies": { - "frac": "~1.1.2" - }, - "engines": { - "node": ">=0.8" - } - }, "node_modules/stack-utils": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", @@ -9676,22 +9618,6 @@ "string-width": "^1.0.2 || 2 || 3 || 4" } }, - "node_modules/wmf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz", - "integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==", - "engines": { - "node": ">=0.8" - } - }, - "node_modules/word": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz", - "integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==", - "engines": { - "node": ">=0.8" - } - }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", @@ -9778,18 +9704,9 @@ } }, "node_modules/xlsx": { - "version": "0.18.5", - "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz", - "integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==", - "dependencies": { - "adler-32": "~1.3.0", - "cfb": "~1.2.1", - "codepage": "~1.15.0", - "crc-32": "~1.2.1", - "ssf": "~0.11.2", - "wmf": "~1.0.1", - "word": "~0.3.0" - }, + "version": "0.20.3", + "resolved": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", + "integrity": "sha512-oLDq3jw7AcLqKWH2AhCpVTZl8mf6X2YReP+Neh0SJUzV/BdZYjth94tG5toiMB1PPrYtxOCfaoUCkvtuH+3AJA==", "bin": { "xlsx": "bin/xlsx.njs" }, diff --git a/package.json b/package.json index d0ed697..3803e74 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "rxjs": "^7.8.1", "swagger-ui-express": "^5.0.1", "uuid": "^10.0.0", - "xlsx": "^0.18.5" + "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz" }, "devDependencies": { "@nestjs/cli": "^10.0.0", diff --git a/src/dto/schedule.dto.ts b/src/dto/schedule.dto.ts index 0d78c40..36faa94 100644 --- a/src/dto/schedule.dto.ts +++ b/src/dto/schedule.dto.ts @@ -4,6 +4,7 @@ import { IsBoolean, IsDate, IsEnum, + IsHash, IsNumber, IsObject, IsOptional, @@ -204,6 +205,22 @@ export class GroupDto { } } +export class CacheStatusDto { + @ApiProperty({ + example: true, + description: "Нужно ли обновить ссылку для скачивания xls?", + }) + @IsBoolean() + cacheUpdateRequired: boolean; + + @ApiProperty({ + example: "e6ff169b01608addf998dbf8f40b019a7f514239", + description: "Хеш последних полученных данных", + }) + @IsHash("sha1") + cacheHash: string; +} + export class ScheduleDto { @ApiProperty({ example: new Date(), @@ -213,13 +230,6 @@ export class ScheduleDto { @IsDate() updatedAt: Date; - @ApiProperty({ - example: '"66d88751-1b800"', - description: "ETag файла с расписанием на сервере политехникума", - }) - @IsString() - etag: string; - @ApiProperty({ description: "Расписание групп" }) @IsObject() @IsOptional() @@ -242,14 +252,6 @@ export class ScheduleDto { }) @Type(() => Object) lastChangedDays: Array>; - - @ApiProperty({ - example: false, - description: - "Требуется ли пользовательское обновление ссылки для скачивания расписания", - }) - @IsBoolean() - updateRequired: boolean; } export class GroupScheduleRequestDto extends PickType(GroupDto, ["name"]) {} @@ -279,20 +281,12 @@ export class GroupScheduleDto extends OmitType(ScheduleDto, [ @ValidateNested({ each: true }) @Type(() => Number) lastChangedDays: Array; - - @ApiProperty({ - example: false, - description: - "Требуется ли пользовательское обновление ссылки для скачивания расписания", - }) - @IsBoolean() - updateRequired: boolean; } export class SiteMainPageDto { @ApiProperty({ - example: "
", - description: "Код страницы политехникума для скачивания", + example: "MHz=", + description: "Страница политехникума", }) @IsBase64() mainPage: string; diff --git a/src/schedule/internal/schedule-parser/schedule-parser.ts b/src/schedule/internal/schedule-parser/schedule-parser.ts index 6b1755a..83bd244 100644 --- a/src/schedule/internal/schedule-parser/schedule-parser.ts +++ b/src/schedule/internal/schedule-parser/schedule-parser.ts @@ -14,7 +14,7 @@ import { import { toNormalString, trimAll } from "../../../utility/string.util"; type InternalId = { row: number; column: number; name: string }; -type InternalDay = InternalId & { lessons: Array }; +type InternalDay = InternalId; export class ScheduleParseResult { etag: string; @@ -107,7 +107,17 @@ export class ScheduleParser { ++row; } - days.push({ row: row, column: 0, name: dayName, lessons: [] }); + const dayMonthIdx = /[А-Яа-я]+\s(\d+)\.\d+\.\d+/.exec( + trimAll(dayName), + ); + + if (dayMonthIdx === null) continue; + + days.push({ + row: row, + column: 0, + name: dayName, + }); if ( days.length > 2 && @@ -130,15 +140,6 @@ export class ScheduleParser { const downloadData = await this.xlsDownloader.downloadXLS(); - if (downloadData.updateRequired && downloadData.etag.length === 0) { - return { - updateRequired: true, - groups: [], - etag: "", - affectedDays: [], - }; - } - if ( !downloadData.new && this.lastResult && @@ -152,8 +153,6 @@ export class ScheduleParser { return this.lastResult; } - console.debug("Чтение кешированного XLS документа..."); - const workBook = XLSX.read(downloadData.fileData); const workSheet = workBook.Sheets[workBook.SheetNames[0]]; diff --git a/src/schedule/internal/xls-downloader/basic-xls-downloader.ts b/src/schedule/internal/xls-downloader/basic-xls-downloader.ts index f910c7e..b0034e0 100644 --- a/src/schedule/internal/xls-downloader/basic-xls-downloader.ts +++ b/src/schedule/internal/xls-downloader/basic-xls-downloader.ts @@ -5,7 +5,10 @@ import { } from "./xls-downloader.base"; import axios from "axios"; import { JSDOM } from "jsdom"; -import { NotAcceptableException } from "@nestjs/common"; +import { + NotAcceptableException, + ServiceUnavailableException, +} from "@nestjs/common"; export class BasicXlsDownloader extends XlsDownloaderBase { cache: XlsDownloaderResult | null = null; @@ -85,13 +88,9 @@ export class BasicXlsDownloader extends XlsDownloaderBase { return this.getCachedXLS(); if (!this.preparedData) { - return { - updateRequired: true, - etag: "", - new: true, - fileData: new ArrayBuffer(1), - updateDate: "", - }; + throw new ServiceUnavailableException( + "Отсутствует начальная ссылка на скачивание!", + ); } // noinspection Annotator diff --git a/src/schedule/schedule.controller.ts b/src/schedule/schedule.controller.ts index 5d3a1bd..6315724 100644 --- a/src/schedule/schedule.controller.ts +++ b/src/schedule/schedule.controller.ts @@ -10,6 +10,7 @@ import { import { AuthGuard } from "../auth/auth.guard"; import { ScheduleService } from "./schedule.service"; import { + CacheStatusDto, GroupScheduleDto, GroupScheduleRequestDto, ScheduleDto, @@ -97,4 +98,17 @@ export class ScheduleController { ): Promise { return await this.scheduleService.updateSiteMainPage(siteMainPageDto); } + + @ApiExtraModels(CacheStatusDto) + @ApiOperation({ + summary: "Получение информации о кеше", + tags: ["schedule", "cache"], + }) + @ApiOkResponse({ description: "Получение данных прошло успешно" }) + @ResultDto(CacheStatusDto) + @HttpCode(HttpStatus.OK) + @Get("cache-status") + getCacheStatus(): CacheStatusDto { + return this.scheduleService.getCacheStatus(); + } } diff --git a/src/schedule/schedule.service.ts b/src/schedule/schedule.service.ts index 58bbb99..c9267d9 100644 --- a/src/schedule/schedule.service.ts +++ b/src/schedule/schedule.service.ts @@ -1,9 +1,4 @@ -import { - Inject, - Injectable, - NotFoundException, - ServiceUnavailableException, -} from "@nestjs/common"; +import { Inject, Injectable, NotFoundException } from "@nestjs/common"; import { ScheduleParser, ScheduleParseResult, @@ -11,6 +6,7 @@ import { import { BasicXlsDownloader } from "./internal/xls-downloader/basic-xls-downloader"; import { XlsDownloaderCacheMode } from "./internal/xls-downloader/xls-downloader.base"; import { + CacheStatusDto, GroupDto, GroupScheduleDto, ScheduleDto, @@ -20,6 +16,7 @@ import { import { Cache, CACHE_MANAGER } from "@nestjs/cache-manager"; import { instanceToPlain } from "class-transformer"; import { cacheGetOrFill } from "../utility/cache.util"; +import * as crypto from "crypto"; @Injectable() export class ScheduleService { @@ -30,24 +27,33 @@ export class ScheduleService { ), ); - private lastCacheUpdate: Date = new Date(0); + private cacheUpdatedAt: Date = new Date(0); + private cacheHash: string = "0000000000000000000000000000000000000000"; + private lastChangedDays: Array> = []; constructor(@Inject(CACHE_MANAGER) private readonly cacheManager: Cache) {} + getCacheStatus(): CacheStatusDto { + return { + cacheHash: this.cacheHash, + cacheUpdateRequired: + (Date.now() - this.cacheUpdatedAt.valueOf()) / 1000 / 60 >= 5, + }; + } + private async getSourceSchedule(): Promise { return cacheGetOrFill(this.cacheManager, "sourceSchedule", async () => { - this.lastCacheUpdate = new Date(); - const schedule = await this.scheduleParser.getSchedule(); schedule.groups = ScheduleService.toObject( schedule.groups, ) as Array; - if (schedule.updateRequired && schedule.etag.length === 0) - throw new ServiceUnavailableException( - "Отсутствует начальная ссылка на скачивание!", - ); + this.cacheUpdatedAt = new Date(); + this.cacheHash = crypto + .createHash("sha1") + .update(schedule.etag) + .digest("hex"); return schedule; }); @@ -62,24 +68,26 @@ export class ScheduleService { } async getSchedule(): Promise { - return cacheGetOrFill(this.cacheManager, "schedule", async () => { - const sourceSchedule = await this.getSourceSchedule(); + return cacheGetOrFill( + this.cacheManager, + "schedule", + async (): Promise => { + const sourceSchedule = await this.getSourceSchedule(); - for (const groupName in sourceSchedule.affectedDays) { - const affectedDays = sourceSchedule.affectedDays[groupName]; + for (const groupName in sourceSchedule.affectedDays) { + const affectedDays = sourceSchedule.affectedDays[groupName]; - if (affectedDays?.length !== 0) - this.lastChangedDays[groupName] = affectedDays; - } + if (affectedDays?.length !== 0) + this.lastChangedDays[groupName] = affectedDays; + } - return { - updatedAt: this.lastCacheUpdate, - groups: ScheduleService.toObject(sourceSchedule.groups), - etag: sourceSchedule.etag, - lastChangedDays: this.lastChangedDays, - updateRequired: sourceSchedule.updateRequired, - }; - }); + return { + updatedAt: this.cacheUpdatedAt, + groups: ScheduleService.toObject(sourceSchedule.groups), + lastChangedDays: this.lastChangedDays, + }; + }, + ); } async getGroup(group: string): Promise { @@ -92,11 +100,9 @@ export class ScheduleService { } return { - updatedAt: this.lastCacheUpdate, + updatedAt: this.cacheUpdatedAt, group: schedule.groups[group], - etag: schedule.etag, lastChangedDays: this.lastChangedDays[group] ?? [], - updateRequired: schedule.updateRequired, }; } @@ -127,5 +133,6 @@ export class ScheduleService { .setPreparedData(siteMainPageDto.mainPage); await this.cacheManager.reset(); + await this.getSourceSchedule(); } }