Merge pull request #10 from stscoundrel/feature/improved-column-detec…

…tion

Columns: detect specific space groups as dividers

parser/src/parser/columns.py

@@ -1,14 +1,42 @@
 from typing import Final
 
 _divider: Final[str] = "|"
+_spaces_divider: Final[str] = "    "
+
+
+def _get_column_divider(line: str) -> str:
+    if _divider in line:
+        # Divider was read in OCR, simple case.
+        return _divider
+
+    # If divider is missing, it was probably misread.
+    # If divider is missing, it generally means there will be a
+    # grouping of spaces where the divider should be.
+    # Four spaces seems common enough case to act as divider.
+    return _spaces_divider
+
+
+def _get_divided_lines(line: str) -> list[str]:
+    divider = _get_column_divider(line)
+    divided = line.split(divider)
+
+    # If we're dealing with spaces divider, we want to preserve
+    # the whitespace after the split. Therefore, append it to
+    # all but the first item in split list.
+    if divider == _spaces_divider:
+        print("APPENDING SPACES")
+        for idx, divided_part in enumerate(divided[1:]):
+            divided[idx + 1] = f"{_spaces_divider}{divided_part}"
+
+    return divided
 
 
 def parse_column(page: list[str]) -> list[str]:
     left_column = []
     right_column = []
 
     for line in page[1:]:  # First line is meta info.
-        divided = line.split(_divider)
+        divided = _get_divided_lines(line)
 
         match len(divided):
             case 1:

parser/tests/test_columns.py

@@ -7,16 +7,20 @@ def test_column_combining() -> None:
         "Lorem ipsum | Country roads",
         "dolor sit amet | take me home",
         "dolor sit igitur | to the place I belong",
+        # Missing divider, to be recognized with group of spaces.
+        "werden. Tavsen. 67; smlgn. Sch. u.     Ablat se oblat.",
     ]
 
     expected = [
         "Meta line about the page",
         "Lorem ipsum ",
         "dolor sit amet ",
         "dolor sit igitur ",
+        "werden. Tavsen. 67; smlgn. Sch. u.",
         " Country roads",
         " take me home",
         " to the place I belong",
+        "     Ablat se oblat.",
     ]
 
     result = columns.parse_column(input)