feat(parser): speed improvement, lesson type guessing and parsing of merged lesson cabinets

Cargo.lock

@@ -1258,15 +1258,6 @@ dependencies = [
  "slab",
 ]
 
-[[package]]
-name = "fuzzy-matcher"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "54614a3312934d066701a80f20f15fa3b56d67ac7722b39eea5b4c9dd1d66c94"
-dependencies = [
- "thread_local",
-]
-
 [[package]]
 name = "gcc"
 version = "0.3.55"
@@ -2881,10 +2872,11 @@ dependencies = [
  "chrono",
  "criterion",
  "derive_more",
- "fuzzy-matcher",
  "regex",
+ "sentry",
  "serde",
  "serde_repr",
+ "strsim",
  "utoipa",
 ]
 
@@ -3383,16 +3375,6 @@ dependencies = [
  "syn 2.0.100",
 ]
 
-[[package]]
-name = "thread_local"
-version = "1.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c"
-dependencies = [
- "cfg-if",
- "once_cell",
-]
-
 [[package]]
 name = "time"
 version = "0.1.45"

schedule-parser/Cargo.toml

@@ -10,11 +10,12 @@ test-utils = []
 calamine = "0.26"
 chrono = { version = "0.4", features = ["serde"] }
 derive_more = { version = "2", features = ["full"] }
+sentry = "0.38"
 serde = { version = "1.0.219", features = ["derive"] }
 serde_repr = "0.1.20"
-fuzzy-matcher = "0.3.7"
 regex = "1.11.1"
 utoipa = { version = "5", features = ["chrono"] }
+strsim = "0.11.1"
 
 [dev-dependencies]
 criterion = "0.6"

schedule-parser/src/lib.rs

@@ -1,15 +1,13 @@
-use crate::schema::internal::{BoundariesCellInfo, DayCellInfo, GroupCellInfo};
+use crate::LessonParseResult::{Lessons, Street};
 use crate::schema::LessonType::Break;
+use crate::schema::internal::{BoundariesCellInfo, DayCellInfo, GroupCellInfo};
 use crate::schema::{
     Day, ErrorCell, ErrorCellPos, Lesson, LessonBoundaries, LessonSubGroup, LessonType, ParseError,
     ParseResult, ScheduleEntry,
 };
 use crate::worksheet::WorkSheet;
-use crate::LessonParseResult::{Lessons, Street};
+use calamine::{Reader, Xls, open_workbook_from_rs};
-use calamine::{open_workbook_from_rs, Reader, Xls};
 use chrono::{DateTime, Duration, NaiveDate, NaiveTime, Utc};
-use fuzzy_matcher::skim::SkimMatcherV2;
-use fuzzy_matcher::FuzzyMatcher;
 use regex::Regex;
 use std::collections::HashMap;
 use std::io::Cursor;
@@ -88,60 +86,29 @@ enum LessonParseResult {
     Street(String),
 }
 
-trait StringInnerSlice {
-    /// Obtaining a line from the line on the initial and final index.
-    fn inner_slice(&self, from: usize, to: usize) -> Self;
-}
-
-impl StringInnerSlice for String {
-    fn inner_slice(&self, from: usize, to: usize) -> Self {
-        self.chars()
-            .take(from)
-            .chain(self.chars().skip(to))
-            .collect()
-    }
-}
-
 // noinspection GrazieInspection
 /// Obtaining a non-standard type of lesson by name.
-fn guess_lesson_type(name: &String) -> Option<(String, LessonType)> {
+fn guess_lesson_type(text: &String) -> Option<LessonType> {
-    let map: HashMap<String, LessonType> = HashMap::from([
+    static MAP: LazyLock<HashMap<&str, LessonType>> = LazyLock::new(|| {
-        ("(консультация)".to_string(), LessonType::Consultation),
+        HashMap::from([
-        (
+            ("консультация", LessonType::Consultation),
-            "самостоятельная работа".to_string(),
+            ("самостоятельная работа", LessonType::IndependentWork),
-            LessonType::IndependentWork,
+            ("зачет", LessonType::Exam),
-        ),
+            ("зачет с оценкой", LessonType::ExamWithGrade),
-        ("зачет".to_string(), LessonType::Exam),
+            ("экзамен", LessonType::ExamDefault),
-        ("зачет с оценкой".to_string(), LessonType::ExamWithGrade),
+        ])
-        ("экзамен".to_string(), LessonType::ExamDefault),
+    });
-    ]);
 
-    let matcher = SkimMatcherV2::default();
+    let name_lower = text.to_lowercase();
-    let name_lower = name.to_lowercase();
 
-    type SearchResult<'a> = (&'a LessonType, i64, Vec<usize>);
+    match MAP
-
-    let mut search_results: Vec<SearchResult> = map
         .iter()
-        .map(|entry| -> SearchResult {
+        .map(|(text, lesson_type)| (lesson_type, strsim::levenshtein(text, &*name_lower)))
-            if let Some((score, indices)) = matcher.fuzzy_indices(&*name_lower, entry.0) {
+        .filter(|x| x.1 <= 4)
-                return (entry.1, score, indices);
+        .min_by_key(|(_, score)| *score)
-            }
+    {
-
+        None => None,
-            (entry.1, 0, Vec::new())
+        Some(v) => Some(v.0.clone()),
-        })
-        .collect();
-    search_results.sort_by(|a, b| b.1.cmp(&a.1));
-
-    let guessed_type = search_results.first().unwrap();
-
-    if guessed_type.1 > 80 {
-        Some((
-            name.inner_slice(guessed_type.2[0], guessed_type.2[guessed_type.2.len() - 1]),
-            guessed_type.0.clone(),
-        ))
-    } else {
-        None
     }
 }
 
@@ -155,28 +122,25 @@ fn parse_lesson(
 ) -> Result<LessonParseResult, ParseError> {
     let row = lesson_boundaries.xls_range.0.0;
 
-    let (name, lesson_type) = {
+    let name = {
-        let full_name = match worksheet.get_string_from_cell(row, group_column) {
+        let cell_data = match worksheet.get_string_from_cell(row, group_column) {
             Some(x) => x,
             None => return Ok(Lessons(Vec::new())),
         };
 
         static OTHER_STREET_RE: LazyLock<Regex> =
-            LazyLock::new(|| Regex::new(r"^[А-Я][а-я]+,?\s?[0-9]+$").unwrap());
+            LazyLock::new(|| Regex::new(r"^[А-Я][а-я]+[,\s]\d+$").unwrap());
 
-        if OTHER_STREET_RE.is_match(&full_name) {
+        if OTHER_STREET_RE.is_match(&cell_data) {
-            return Ok(Street(full_name));
+            return Ok(Street(cell_data));
         }
 
-        match guess_lesson_type(&full_name) {
+        cell_data
-            Some(x) => x,
-            None => (full_name, lesson_boundaries.lesson_type.clone()),
-        }
     };
 
-    let (default_range, lesson_time) = {
     let cell_range = worksheet.get_merge_from_start(row, group_column);
 
+    let (default_range, lesson_time) = {
         let end_time_arr = day_boundaries
             .iter()
             .filter(|time| time.xls_range.1.0 == cell_range.1.0)
@@ -205,10 +169,17 @@ fn parse_lesson(
         Ok((range, time))
     }?;
 
-    let (name, mut subgroups) = parse_name_and_subgroups(&name)?;
+    let (name, mut subgroups, lesson_type) = parse_name_and_subgroups(&name)?;
 
     {
-        let cabinets: Vec<String> = parse_cabinets(worksheet, row, group_column + 1);
+        let cabinets: Vec<String> = parse_cabinets(
+            worksheet,
+            (
+                cell_range.0.0,
+                cell_range.1.0,
+            ),
+            group_column + 1,
+        );
 
         match cabinets.len() {
             // Если кабинетов нет, но есть подгруппы, назначаем им кабинет "??"
@@ -254,7 +225,7 @@ fn parse_lesson(
     };
 
     let lesson = Lesson {
-        lesson_type,
+        lesson_type: lesson_type.unwrap_or(lesson_boundaries.lesson_type.clone()),
         default_range,
         name: Some(name),
         time: lesson_time,
@@ -285,10 +256,12 @@ fn parse_lesson(
 }
 
 /// Obtaining a list of cabinets to the right of the lesson cell.
-fn parse_cabinets(worksheet: &WorkSheet, row: u32, column: u32) -> Vec<String> {
+fn parse_cabinets(worksheet: &WorkSheet, row_range: (u32, u32), column: u32) -> Vec<String> {
     let mut cabinets: Vec<String> = Vec::new();
 
-    if let Some(raw) = worksheet.get_string_from_cell(row, column) {
+    for row in row_range.0..row_range.1 {
+        let raw = or_continue!(worksheet.get_string_from_cell(row, column));
+
         let clean = raw.replace("\n", " ");
         let parts: Vec<&str> = clean.split(" ").collect();
 
@@ -297,58 +270,116 @@ fn parse_cabinets(worksheet: &WorkSheet, row: u32, column: u32) -> Vec<String> {
 
             cabinets.push(clean_part);
         }
+
+        break;
     }
 
     cabinets
 }
 
+//noinspection GrazieInspection
 /// Getting the "pure" name of the lesson and list of teachers from the text of the lesson cell.
-fn parse_name_and_subgroups(name: &String) -> Result<(String, Vec<LessonSubGroup>), ParseError> {
+fn parse_name_and_subgroups(
-    static LESSON_RE: LazyLock<Regex> =
+    text: &String,
-        LazyLock::new(|| Regex::new(r"(?:[А-Я][а-я]+[А-Я]{2}(?:\([0-9][а-я]+\))?)+$").unwrap());
+) -> Result<(String, Vec<LessonSubGroup>, Option<LessonType>), ParseError> {
-    static TEACHER_RE: LazyLock<Regex> =
+    // Части названия пары:
-        LazyLock::new(|| Regex::new(r"([А-Я][а-я]+)([А-Я])([А-Я])(?:\(([0-9])[а-я]+\))?").unwrap());
+    // 1. Само название.
-    static CLEAN_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"[\s.,]+").unwrap());
+    // 2. Список преподавателей и подгрупп.
-    static END_CLEAN_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"[.\s]+$").unwrap());
+    // 3. "Модификатор" (чаще всего).
+    //
+    // Регулярное выражение для получения ФИО преподавателей и номеров подгрупп (aka. второй части).
+    // (?:[А-Я][а-я]+\s?(?:[А-Я][\s.]*){2}(?:\(\d\s?[а-я]+\))?(?:, )?)+[\s.]*
+    //
+    // Подробнее:
+    // (?:
+    //     [А-Я][а-я]+         - Фамилия.
+    //     \s?                 - Кто знает, будет ли там пробел.
+    //     (?:[А-Я][\s.]*){2}  - Имя и отчество с учётом случайных пробелов и точек.
+    //     (?:
+    //         \(              - Открытие подгруппы.
+    //         \s?             - Кто знает, будет ли там пробел.
+    //         \d              - Номер подгруппы.
+    //         \s?             - Кто знает, будет ли там пробел.
+    //         [а-я\s]+        - Слово "подгруппа" с учётов ошибок.
+    //         \)              - Закрытие подгруппы.
+    //     )?                  - Явное указание подгруппы может отсутствовать по понятным причинам.
+    //     (?:, )?             - Разделители между отдельными частями.
+    // )+
+    // [\s.]*                  - Забираем с собой всякий мусор, что бы не передать его в третью часть.
 
-    let (teachers, lesson_name) = {
+    static NAMES_REGEX: LazyLock<Regex> = LazyLock::new(|| {
-        let clean_name = CLEAN_RE.replace_all(&name, "").to_string();
+        Regex::new(
-
+            r"(?:[А-Я][а-я]+\s?(?:[А-Я][\s.]*){2}(?:\(\s*\d\s*[а-я\s]+\))?(?:[\s,]+)?)+[\s.]*",
-        if let Some(captures) = LESSON_RE.captures(&clean_name) {
+        )
-            let capture = captures.get(0).unwrap();
+        .unwrap()
-            let capture_str = capture.as_str().to_string();
+    });
-            let capture_name: String = capture_str.chars().take(5).collect();
+
-
+    // Отчистка
-            (
+    static CLEAN_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"[\s\n\t]+").unwrap());
-                END_CLEAN_RE.replace(&capture_str, "").to_string(),
+
-                END_CLEAN_RE
+    let text = CLEAN_RE
-                    .replace(&name[0..name.find(&*capture_name).unwrap()], "")
+        .replace(&text.replace(&[' ', '\t', '\n'], " "), " ")
-                    .to_string(),
+        .to_string();
+
+    let (lesson_name, mut subgroups, lesson_type) = match NAMES_REGEX.captures(&text) {
+        Some(captures) => {
+            let capture = captures.get(0).unwrap();
+
+            let subgroups: Vec<LessonSubGroup> = {
+                let src = capture.as_str().replace(&[' ', '.'], "");
+
+                src.split(',')
+                    .map(|name| {
+                        let open_bracket_index = name.find('(');
+
+                        let subgroup_number = open_bracket_index.map_or(0, |index| {
+                            name[(index + 1)..(index + 2)].parse::<u8>().unwrap()
+                        });
+
+                        let teacher_name = {
+                            let name_end = open_bracket_index.unwrap_or_else(|| name.len());
+
+                            // Я ебал. Как же я долго до этого доходил.
+                            format!(
+                                "{} {}.{}.",
+                                name.get(..name_end - 4).unwrap(),
+                                name.get(name_end - 4..name_end - 2).unwrap(),
+                                name.get(name_end - 2..name_end).unwrap(),
                             )
-        } else {
-            return Ok((END_CLEAN_RE.replace(&name, "").to_string(), Vec::new()));
-        }
                         };
 
-    let mut subgroups: Vec<LessonSubGroup> = Vec::new();
+                        LessonSubGroup {
-
+                            number: subgroup_number,
-    let teacher_it = TEACHER_RE.captures_iter(&teachers);
-
-    for captures in teacher_it {
-        subgroups.push(LessonSubGroup {
-            number: match captures.get(4) {
-                Some(capture) => capture.as_str().to_string().parse::<u8>().unwrap(),
-                None => 0,
-            },
                             cabinet: None,
-            teacher: format!(
+                            teacher: teacher_name,
-                "{} {}.{}.",
-                captures.get(1).unwrap().as_str().to_string(),
-                captures.get(2).unwrap().as_str().to_string(),
-                captures.get(3).unwrap().as_str().to_string()
-            ),
-        });
                         }
+                    })
+                    .collect()
+            };
+
+            let name = text[..capture.start()].trim().to_string();
+            let extra = text[capture.end()..].trim().to_string();
+
+            let lesson_type = if extra.len() > 4 {
+                let result = guess_lesson_type(&extra);
+
+                #[cfg(not(debug_assertions))]
+                if result.is_none() {
+                    sentry::capture_message(
+                        &*format!("Не удалось угадать тип пары '{}'!", extra),
+                        sentry::Level::Warning,
+                    );
+                }
+
+                result
+            } else {
+                None
+            };
+
+            (name, subgroups, lesson_type)
+        }
+        None => (text, Vec::new(), None),
+    };
 
     // фикс, если у кого-то отсутствует индекс подгруппы
 
@@ -384,7 +415,7 @@ fn parse_name_and_subgroups(name: &String) -> Result<(String, Vec<LessonSubGroup
         subgroups.reverse()
     }
 
-    Ok((lesson_name, subgroups))
+    Ok((lesson_name, subgroups, lesson_type))
 }
 
 /// Getting the start and end of a pair from a cell in the first column of a document.
@@ -719,9 +750,17 @@ pub mod tests {
         assert!(result.groups.contains_key("ИС-214/23"));
 
         let group = result.groups.get("ИС-214/23").unwrap();
-        let thursday = group.days.get(3).unwrap();
 
+        let thursday = group.days.get(3).unwrap();
         assert_eq!(thursday.lessons.len(), 1);
-        assert_eq!(thursday.lessons[0].default_range.unwrap()[1], 3);
+
+        let lesson = &thursday.lessons[0];
+        assert_eq!(lesson.default_range.unwrap()[1], 3);
+        assert!(lesson.subgroups.is_some());
+
+        let subgroups = lesson.subgroups.as_ref().unwrap();
+        assert_eq!(subgroups.len(), 2);
+        assert_eq!(subgroups[0].cabinet, Some("44".to_string()));
+        assert_eq!(subgroups[1].cabinet, Some("43".to_string()));
     }
 }