Обновлённая система кеширования и чистка кода.

2025-12-06 09:47:46 +03:00 · 2024-09-26 01:37:44 +04:00
parent 811feff173
commit cd9dc319eb
7 changed files with 95 additions and 165 deletions
--- a/package-lock.json
+++ b/package-lock.json
@@ -28,7 +28,7 @@
        "rxjs": "^7.8.1",
        "swagger-ui-express": "^5.0.1",
        "uuid": "^10.0.0",
-        "xlsx": "^0.18.5"
+        "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz"
      },
      "devDependencies": {
        "@nestjs/cli": "^10.0.0",
@@ -2798,14 +2798,6 @@
        "node": ">=0.4.0"
      }
    },
-    "node_modules/adler-32": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz",
-      "integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==",
-      "engines": {
-        "node": ">=0.8"
-      }
-    },
    "node_modules/agent-base": {
      "version": "6.0.2",
      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
@@ -3458,18 +3450,6 @@
        }
      ]
    },
-    "node_modules/cfb": {
-      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz",
-      "integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==",
-      "dependencies": {
-        "adler-32": "~1.3.0",
-        "crc-32": "~1.2.0"
-      },
-      "engines": {
-        "node": ">=0.8"
-      }
-    },
    "node_modules/chalk": {
      "version": "4.1.2",
      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
@@ -3675,14 +3655,6 @@
        "node": ">= 0.12.0"
      }
    },
-    "node_modules/codepage": {
-      "version": "1.15.0",
-      "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz",
-      "integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==",
-      "engines": {
-        "node": ">=0.8"
-      }
-    },
    "node_modules/collect-v8-coverage": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.2.tgz",
@@ -3874,17 +3846,6 @@
        }
      }
    },
-    "node_modules/crc-32": {
-      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz",
-      "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==",
-      "bin": {
-        "crc32": "bin/crc32.njs"
-      },
-      "engines": {
-        "node": ">=0.8"
-      }
-    },
    "node_modules/create-jest": {
      "version": "29.7.0",
      "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz",
@@ -5030,14 +4991,6 @@
        "node": ">= 0.6"
      }
    },
-    "node_modules/frac": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz",
-      "integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==",
-      "engines": {
-        "node": ">=0.8"
-      }
-    },
    "node_modules/fresh": {
      "version": "0.5.2",
      "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
@@ -8550,17 +8503,6 @@
      "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==",
      "dev": true
    },
-    "node_modules/ssf": {
-      "version": "0.11.2",
-      "resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz",
-      "integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==",
-      "dependencies": {
-        "frac": "~1.1.2"
-      },
-      "engines": {
-        "node": ">=0.8"
-      }
-    },
    "node_modules/stack-utils": {
      "version": "2.0.6",
      "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz",
@@ -9676,22 +9618,6 @@
        "string-width": "^1.0.2 || 2 || 3 || 4"
      }
    },
-    "node_modules/wmf": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz",
-      "integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==",
-      "engines": {
-        "node": ">=0.8"
-      }
-    },
-    "node_modules/word": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz",
-      "integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==",
-      "engines": {
-        "node": ">=0.8"
-      }
-    },
    "node_modules/word-wrap": {
      "version": "1.2.5",
      "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz",
@@ -9778,18 +9704,9 @@
      }
    },
    "node_modules/xlsx": {
-      "version": "0.18.5",
-      "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz",
-      "integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==",
-      "dependencies": {
-        "adler-32": "~1.3.0",
-        "cfb": "~1.2.1",
-        "codepage": "~1.15.0",
-        "crc-32": "~1.2.1",
-        "ssf": "~0.11.2",
-        "wmf": "~1.0.1",
-        "word": "~0.3.0"
-      },
+      "version": "0.20.3",
+      "resolved": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz",
+      "integrity": "sha512-oLDq3jw7AcLqKWH2AhCpVTZl8mf6X2YReP+Neh0SJUzV/BdZYjth94tG5toiMB1PPrYtxOCfaoUCkvtuH+3AJA==",
      "bin": {
        "xlsx": "bin/xlsx.njs"
      },
--- a/package.json
+++ b/package.json
@@ -39,7 +39,7 @@
    "rxjs": "^7.8.1",
    "swagger-ui-express": "^5.0.1",
    "uuid": "^10.0.0",
-    "xlsx": "^0.18.5"
+    "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz"
  },
  "devDependencies": {
    "@nestjs/cli": "^10.0.0",
--- a/src/dto/schedule.dto.ts
+++ b/src/dto/schedule.dto.ts
@@ -4,6 +4,7 @@ import {
 	IsBoolean,
 	IsDate,
 	IsEnum,
+	IsHash,
 	IsNumber,
 	IsObject,
 	IsOptional,
@@ -204,6 +205,22 @@ export class GroupDto {
 	}
 }

+export class CacheStatusDto {
+	@ApiProperty({
+		example: true,
+		description: "Нужно ли обновить ссылку для скачивания xls?",
+	})
+	@IsBoolean()
+	cacheUpdateRequired: boolean;
+
+	@ApiProperty({
+		example: "e6ff169b01608addf998dbf8f40b019a7f514239",
+		description: "Хеш последних полученных данных",
+	})
+	@IsHash("sha1")
+	cacheHash: string;
+}
+
 export class ScheduleDto {
 	@ApiProperty({
 		example: new Date(),
@@ -213,13 +230,6 @@ export class ScheduleDto {
 	@IsDate()
 	updatedAt: Date;

-	@ApiProperty({
-		example: '"66d88751-1b800"',
-		description: "ETag файла с расписанием на сервере политехникума",
-	})
-	@IsString()
-	etag: string;
-
 	@ApiProperty({ description: "Расписание групп" })
 	@IsObject()
 	@IsOptional()
@@ -242,14 +252,6 @@ export class ScheduleDto {
 	})
 	@Type(() => Object)
 	lastChangedDays: Array<Array<number>>;
-
-	@ApiProperty({
-		example: false,
-		description:
-			"Требуется ли пользовательское обновление ссылки для скачивания расписания",
-	})
-	@IsBoolean()
-	updateRequired: boolean;
 }

 export class GroupScheduleRequestDto extends PickType(GroupDto, ["name"]) {}
@@ -279,20 +281,12 @@ export class GroupScheduleDto extends OmitType(ScheduleDto, [
 	@ValidateNested({ each: true })
 	@Type(() => Number)
 	lastChangedDays: Array<number>;
-
-	@ApiProperty({
-		example: false,
-		description:
-			"Требуется ли пользовательское обновление ссылки для скачивания расписания",
-	})
-	@IsBoolean()
-	updateRequired: boolean;
 }

 export class SiteMainPageDto {
 	@ApiProperty({
-		example: "<div></div>",
-		description: "Код страницы политехникума для скачивания",
+		example: "MHz=",
+		description: "Страница политехникума",
 	})
 	@IsBase64()
 	mainPage: string;
--- a/src/schedule/internal/schedule-parser/schedule-parser.ts
+++ b/src/schedule/internal/schedule-parser/schedule-parser.ts
@@ -14,7 +14,7 @@ import {
 import { toNormalString, trimAll } from "../../../utility/string.util";

 type InternalId = { row: number; column: number; name: string };
-type InternalDay = InternalId & { lessons: Array<InternalId> };
+type InternalDay = InternalId;

 export class ScheduleParseResult {
 	etag: string;
@@ -107,7 +107,17 @@ export class ScheduleParser {
 				++row;
 			}

-			days.push({ row: row, column: 0, name: dayName, lessons: [] });
+			const dayMonthIdx = /[А-Яа-я]+\s(\d+)\.\d+\.\d+/.exec(
+				trimAll(dayName),
+			);
+
+			if (dayMonthIdx === null) continue;
+
+			days.push({
+				row: row,
+				column: 0,
+				name: dayName,
+			});

 			if (
 				days.length > 2 &&
@@ -130,15 +140,6 @@ export class ScheduleParser {

 		const downloadData = await this.xlsDownloader.downloadXLS();

-		if (downloadData.updateRequired && downloadData.etag.length === 0) {
-			return {
-				updateRequired: true,
-				groups: [],
-				etag: "",
-				affectedDays: [],
-			};
-		}
-
 		if (
 			!downloadData.new &&
 			this.lastResult &&
@@ -152,8 +153,6 @@ export class ScheduleParser {
 			return this.lastResult;
 		}

-		console.debug("Чтение кешированного XLS документа...");
-
 		const workBook = XLSX.read(downloadData.fileData);
 		const workSheet = workBook.Sheets[workBook.SheetNames[0]];

--- a/src/schedule/internal/xls-downloader/basic-xls-downloader.ts
+++ b/src/schedule/internal/xls-downloader/basic-xls-downloader.ts
@@ -5,7 +5,10 @@ import {
 } from "./xls-downloader.base";
 import axios from "axios";
 import { JSDOM } from "jsdom";
-import { NotAcceptableException } from "@nestjs/common";
+import {
+	NotAcceptableException,
+	ServiceUnavailableException,
+} from "@nestjs/common";

 export class BasicXlsDownloader extends XlsDownloaderBase {
 	cache: XlsDownloaderResult | null = null;
@@ -85,13 +88,9 @@ export class BasicXlsDownloader extends XlsDownloaderBase {
 			return this.getCachedXLS();

 		if (!this.preparedData) {
-			return {
-				updateRequired: true,
-				etag: "",
-				new: true,
-				fileData: new ArrayBuffer(1),
-				updateDate: "",
-			};
+			throw new ServiceUnavailableException(
+				"Отсутствует начальная ссылка на скачивание!",
+			);
 		}

 		// noinspection Annotator
--- a/src/schedule/schedule.controller.ts
+++ b/src/schedule/schedule.controller.ts
@@ -10,6 +10,7 @@ import {
 import { AuthGuard } from "../auth/auth.guard";
 import { ScheduleService } from "./schedule.service";
 import {
+	CacheStatusDto,
 	GroupScheduleDto,
 	GroupScheduleRequestDto,
 	ScheduleDto,
@@ -97,4 +98,17 @@ export class ScheduleController {
 	): Promise<void> {
 		return await this.scheduleService.updateSiteMainPage(siteMainPageDto);
 	}
+
+	@ApiExtraModels(CacheStatusDto)
+	@ApiOperation({
+		summary: "Получение информации о кеше",
+		tags: ["schedule", "cache"],
+	})
+	@ApiOkResponse({ description: "Получение данных прошло успешно" })
+	@ResultDto(CacheStatusDto)
+	@HttpCode(HttpStatus.OK)
+	@Get("cache-status")
+	getCacheStatus(): CacheStatusDto {
+		return this.scheduleService.getCacheStatus();
+	}
 }
--- a/src/schedule/schedule.service.ts
+++ b/src/schedule/schedule.service.ts
@@ -1,9 +1,4 @@
-import {
-	Inject,
-	Injectable,
-	NotFoundException,
-	ServiceUnavailableException,
-} from "@nestjs/common";
+import { Inject, Injectable, NotFoundException } from "@nestjs/common";
 import {
 	ScheduleParser,
 	ScheduleParseResult,
@@ -11,6 +6,7 @@ import {
 import { BasicXlsDownloader } from "./internal/xls-downloader/basic-xls-downloader";
 import { XlsDownloaderCacheMode } from "./internal/xls-downloader/xls-downloader.base";
 import {
+	CacheStatusDto,
 	GroupDto,
 	GroupScheduleDto,
 	ScheduleDto,
@@ -20,6 +16,7 @@ import {
 import { Cache, CACHE_MANAGER } from "@nestjs/cache-manager";
 import { instanceToPlain } from "class-transformer";
 import { cacheGetOrFill } from "../utility/cache.util";
+import * as crypto from "crypto";

@Injectable()
 export class ScheduleService {
@@ -30,24 +27,33 @@ export class ScheduleService {
 		),
 	);

-	private lastCacheUpdate: Date = new Date(0);
+	private cacheUpdatedAt: Date = new Date(0);
+	private cacheHash: string = "0000000000000000000000000000000000000000";
+
 	private lastChangedDays: Array<Array<number>> = [];

 	constructor(@Inject(CACHE_MANAGER) private readonly cacheManager: Cache) {}

+	getCacheStatus(): CacheStatusDto {
+		return {
+			cacheHash: this.cacheHash,
+			cacheUpdateRequired:
+				(Date.now() - this.cacheUpdatedAt.valueOf()) / 1000 / 60 >= 5,
+		};
+	}
+
 	private async getSourceSchedule(): Promise<ScheduleParseResult> {
 		return cacheGetOrFill(this.cacheManager, "sourceSchedule", async () => {
-			this.lastCacheUpdate = new Date();
-
 			const schedule = await this.scheduleParser.getSchedule();
 			schedule.groups = ScheduleService.toObject(
 				schedule.groups,
 			) as Array<GroupDto>;

-			if (schedule.updateRequired && schedule.etag.length === 0)
-				throw new ServiceUnavailableException(
-					"Отсутствует начальная ссылка на скачивание!",
-				);
+			this.cacheUpdatedAt = new Date();
+			this.cacheHash = crypto
+				.createHash("sha1")
+				.update(schedule.etag)
+				.digest("hex");

 			return schedule;
 		});
@@ -62,24 +68,26 @@ export class ScheduleService {
 	}

 	async getSchedule(): Promise<ScheduleDto> {
-		return cacheGetOrFill(this.cacheManager, "schedule", async () => {
-			const sourceSchedule = await this.getSourceSchedule();
+		return cacheGetOrFill(
+			this.cacheManager,
+			"schedule",
+			async (): Promise<ScheduleDto> => {
+				const sourceSchedule = await this.getSourceSchedule();

-			for (const groupName in sourceSchedule.affectedDays) {
-				const affectedDays = sourceSchedule.affectedDays[groupName];
+				for (const groupName in sourceSchedule.affectedDays) {
+					const affectedDays = sourceSchedule.affectedDays[groupName];

-				if (affectedDays?.length !== 0)
-					this.lastChangedDays[groupName] = affectedDays;
-			}
+					if (affectedDays?.length !== 0)
+						this.lastChangedDays[groupName] = affectedDays;
+				}

-			return {
-				updatedAt: this.lastCacheUpdate,
-				groups: ScheduleService.toObject(sourceSchedule.groups),
-				etag: sourceSchedule.etag,
-				lastChangedDays: this.lastChangedDays,
-				updateRequired: sourceSchedule.updateRequired,
-			};
-		});
+				return {
+					updatedAt: this.cacheUpdatedAt,
+					groups: ScheduleService.toObject(sourceSchedule.groups),
+					lastChangedDays: this.lastChangedDays,
+				};
+			},
+		);
 	}

 	async getGroup(group: string): Promise<GroupScheduleDto> {
@@ -92,11 +100,9 @@ export class ScheduleService {
 		}

 		return {
-			updatedAt: this.lastCacheUpdate,
+			updatedAt: this.cacheUpdatedAt,
 			group: schedule.groups[group],
-			etag: schedule.etag,
 			lastChangedDays: this.lastChangedDays[group] ?? [],
-			updateRequired: schedule.updateRequired,
 		};
 	}

@@ -127,5 +133,6 @@ export class ScheduleService {
 			.setPreparedData(siteMainPageDto.mainPage);

 		await this.cacheManager.reset();
+		await this.getSourceSchedule();
 	}
 }