FOSSredder 0.5.0
Loading...
Searching...
No Matches
core::application::importing::internal Namespace Reference

Classes

struct  ColumnGuess
 Stores the inferred column positions for an OCR page. More...
class  DefaultStatementParser
struct  FinalizeStats
struct  OcrLine
 Represents one OCR line with its bounding box and recognized text. More...
struct  PageWork
struct  ParserConfig
 Runtime configuration for parser heuristics and thresholds. More...
struct  RawLineLite
 Lightweight raw-line representation used during grouping and splitting. More...
struct  TransactionMainRow
 Represents the parsed main row of a transaction block. More...

Typedefs

using ProgressReporter = std::function<void(double, const std::string&)>

Functions

bool hasShortDateToken (const std::string &text) noexcept
 Checks whether the text contains a short date token.
bool containsShortDate (const std::string &text) noexcept
 Checks whether the text contains any short date pattern.
std::optional< std::string > findFirstFullDate (const std::string &text) noexcept
 Finds the first full date in a text fragment.
std::vector< size_t > findAmountTokenIndices (const core::application::importing::transaction::internal::OcrLine &line, int valutaX=-1, int bandPx=0) noexcept
 Finds the indices of amount-like tokens in an OCR line.
std::string normalizeAlnumLower (const std::string &s) noexcept
 Normalizes alphanumeric text to lower-case ASCII for matching.
std::optional< int > findTokenCenterX (const core::application::importing::transaction::internal::OcrLine &line, const std::string &tokenLower) noexcept
 Locates the horizontal center of a token inside an OCR line.
std::optional< int > findPhraseCenterX (const core::application::importing::transaction::internal::OcrLine &line, std::initializer_list< const char * > phraseLower) noexcept
 Locates the horizontal center of a multi-token phrase in an OCR line.
bool hasTokenNearX (const core::application::importing::transaction::internal::OcrLine &line, int x, int bandPx) noexcept
 Checks whether any token in the line is close to a horizontal position.
bool hasAmountLikeTokenInLine (const core::application::importing::transaction::internal::OcrLine &line, int valutaX) noexcept
 Checks whether the line contains amount-like tokens near the valuta column.
bool hasLeftDescriptiveText (const core::application::importing::transaction::internal::OcrLine &line, int valutaX) noexcept
 Checks whether the line contains descriptive text to the left of the valuta column.
bool hasAmountNearValuta (const core::application::importing::transaction::internal::OcrLine &line, int valutaX, int bandPx) noexcept
 Checks whether an amount appears close to the valuta reference.
bool isLooseTransactionLine (const core::application::importing::transaction::internal::OcrLine &line, int valutaX) noexcept
 Checks whether the line looks like a loosely formatted transaction row.
std::optional< double > findAndParseAmountInLine (const core::application::importing::transaction::internal::OcrLine &line, int valutaX=-1, std::vector< std::string > *debugOut=nullptr) noexcept
 Finds and parses an amount token inside an OCR line.
std::vector< core::application::importing::transaction::internal::OcrLine > buildOcrLinesFromWords (const std::vector< core::ports::text_recognition::Word > &words)
 Builds OCR lines from raw Tesseract word output.
ColumnGuess inferColumnModelFromLines (const std::vector< core::application::importing::transaction::internal::OcrLine > &lines, size_t scanN=16) noexcept
 Infers a column layout from a set of OCR lines.
std::vector< RawLineLite > groupMergeLinesRaw (const std::vector< RawLineLite > &lines, int maxGapPx)
 Merges raw lines using a gap-based heuristic.
core::application::importing::transaction::internal::OcrLine toOcrLineFromRawWords (const RawLineLite &src, size_t i0, size_t i1) noexcept
 Converts a range of raw words into one OCR line.
core::application::importing::transaction::internal::TransactionMainRow splitMainRowFromRaw (const RawLineLite &src, int valutaX, int debitX, int creditX) noexcept
 Splits a raw line into the main transaction row representation.
core::application::importing::transaction::internal::TransactionMainRow splitMainRowFromOcrLine (const core::application::importing::transaction::internal::OcrLine &src, int valutaX, int debitX, int creditX) noexcept
 Splits an OCR line into the main transaction row representation.
bool isHeaderNoiseLine (const std::string &line)
 Detects whether a line looks like header noise.
bool isFooterLine (const std::string &line)
 Detects whether a line marks a footer section.
bool isPostTransactionFootnote (const std::string &line)
 Detects whether a line belongs to the post-transaction footnote area.
bool isTransactionsSectionHeader (const std::string &line)
 Detects whether a line marks the start of the transactions section.
bool isDebitCreditHeaderLine (const std::string &line)
 Detects whether a line looks like a debit/credit table header.
core::ports::text_recognition::Settings buildRecognitionSettings (int defaultPsm)
core::ports::document_image_processing::MaskRequest buildMaskRequest (const std::vector< uint8_t > &pageBytes, size_t pageIndex, const ImportRequest &req, const core::ports::pdf_rendering::ExtractResult &extractRes)
core::ports::text_recognition::ExtractRequest buildMaskOcrRequest (const std::vector< uint8_t > &pageBytes, const ImportRequest &req)
core::ports::document_image_processing::DetectRequest buildDetectRequest (const std::vector< uint8_t > &maskedBytes, size_t pageIndex, const ImportRequest &req)
core::ports::document_image_processing::CropRequest buildCropRequest (const std::vector< uint8_t > &pageBytes, size_t pageIndex, const ImportRequest &req, const core::ports::document_image_processing::DetectResult &detectResponse)
core::ports::text_recognition::ExtractRequest buildTableOcrRequest (const std::vector< uint8_t > &croppedBytes, size_t pageIndex, const ImportRequest &req, const core::ports::document_image_processing::DetectResult &detectResponse)
std::vector< uint8_t > readImportBytes (const std::filesystem::path &path)
std::vector< uint8_t > readImportBytes (const std::filesystem::path &path, core::ports::diagnostics::IErrorReporter *errorReporter)
PageWork processImportPage (size_t pageIndex, size_t totalPages, const ImportRequest &req, const poppler::RenderResult &renderRes, const poppler::ExtractResult &extractRes, const std::shared_ptr< core::ports::document_image_processing::IDocumentImageProcessor > &documentImageProcessor, const std::shared_ptr< core::ports::text_recognition::ITextRecognizer > &tesseract, core::jobs::SlotLimiter *ocrLimiter, const ProgressReporter &report, std::atomic< size_t > &doneUnits, size_t totalUnits, core::ports::diagnostics::IErrorReporter *errorReporter, ImportResult &out, std::mutex &artifactsMutex)
FinalizeStats finalizeParsedPages (const ImportRequest &req, const std::vector< PageWork > &pages, const std::shared_ptr< core::ports::document_image_processing::IDocumentImageProcessor > &documentImageProcessor, ImportResult &out, std::vector< core::application::importing::draft::TransactionDraft > &all, std::string &carriedBookingDate, int &nextTxIndex, core::ports::diagnostics::IErrorReporter *errorReporter, const ProgressReporter &report, std::mutex &artifactsMutex)
PageWork processImportPage (size_t pageIndex, size_t totalPages, const ImportRequest &req, const core::ports::pdf_rendering::RenderResult &renderRes, const core::ports::pdf_rendering::ExtractResult &extractRes, const std::shared_ptr< core::ports::document_image_processing::IDocumentImageProcessor > &documentImageProcessor, const std::shared_ptr< core::ports::text_recognition::ITextRecognizer > &tesseract, core::jobs::SlotLimiter *ocrLimiter, const ProgressReporter &report, std::atomic< size_t > &doneUnits, size_t totalUnits, core::ports::diagnostics::IErrorReporter *errorReporter, ImportResult &out, std::mutex &artifactsMutex)
ColumnGuess inferColumnModelFromLines (const std::vector< OcrLine > &lines, size_t scanN) noexcept
static bool containsAny (const std::string &hayLower, std::initializer_list< const char * > needlesLower)

Variables

ParserConfig parserConfig
 Shared parser configuration instance used by import parsing helpers.

Typedef Documentation

◆ ProgressReporter

using core::application::importing::internal::ProgressReporter = std::function<void(double, const std::string&)>

Definition at line 29 of file ImportPipelineHelpers.h.

Function Documentation

◆ buildCropRequest()

core::ports::document_image_processing::CropRequest core::application::importing::internal::buildCropRequest ( const std::vector< uint8_t > & pageBytes,
size_t pageIndex,
const ImportRequest & req,
const core::ports::document_image_processing::DetectResult & detectResponse )
inline

Definition at line 83 of file ImportPageRequests.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ buildDetectRequest()

core::ports::document_image_processing::DetectRequest core::application::importing::internal::buildDetectRequest ( const std::vector< uint8_t > & maskedBytes,
size_t pageIndex,
const ImportRequest & req )
inline

Definition at line 70 of file ImportPageRequests.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ buildMaskOcrRequest()

core::ports::text_recognition::ExtractRequest core::application::importing::internal::buildMaskOcrRequest ( const std::vector< uint8_t > & pageBytes,
const ImportRequest & req )
inline

Definition at line 62 of file ImportPageRequests.h.

Here is the caller graph for this function:

◆ buildMaskRequest()

core::ports::document_image_processing::MaskRequest core::application::importing::internal::buildMaskRequest ( const std::vector< uint8_t > & pageBytes,
size_t pageIndex,
const ImportRequest & req,
const core::ports::pdf_rendering::ExtractResult & extractRes )
inline

Definition at line 29 of file ImportPageRequests.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ buildOcrLinesFromWords()

std::vector< OcrLine > core::application::importing::internal::buildOcrLinesFromWords ( const std::vector< core::ports::text_recognition::Word > & words)

Builds OCR lines from raw Tesseract word output.

Parameters
wordsOCR words to convert.
Returns
Constructed OCR lines.

Definition at line 373 of file ParserHelpers.cpp.

Here is the call graph for this function:

◆ buildRecognitionSettings()

core::ports::text_recognition::Settings core::application::importing::internal::buildRecognitionSettings ( int defaultPsm)
inline

Definition at line 22 of file ImportPageRequests.h.

◆ buildTableOcrRequest()

core::ports::text_recognition::ExtractRequest core::application::importing::internal::buildTableOcrRequest ( const std::vector< uint8_t > & croppedBytes,
size_t pageIndex,
const ImportRequest & req,
const core::ports::document_image_processing::DetectResult & detectResponse )
inline

Definition at line 97 of file ImportPageRequests.h.

Here is the caller graph for this function:

◆ containsAny()

bool core::application::importing::internal::containsAny ( const std::string & hayLower,
std::initializer_list< const char * > needlesLower )
static

Definition at line 97 of file ParserHeuristics.cpp.

Here is the caller graph for this function:

◆ containsShortDate()

bool core::application::importing::internal::containsShortDate ( const std::string & text)
noexcept

Checks whether the text contains any short date pattern.

Parameters
textInput text to inspect.
Returns
True when a short date is present.

Definition at line 29 of file ParserDateUtils.cpp.

Here is the call graph for this function:

◆ finalizeParsedPages()

FinalizeStats core::application::importing::internal::finalizeParsedPages ( const ImportRequest & req,
const std::vector< PageWork > & pages,
const std::shared_ptr< core::ports::document_image_processing::IDocumentImageProcessor > & documentImageProcessor,
ImportResult & out,
std::vector< core::application::importing::draft::TransactionDraft > & all,
std::string & carriedBookingDate,
int & nextTxIndex,
core::ports::diagnostics::IErrorReporter * errorReporter,
const ProgressReporter & report,
std::mutex & artifactsMutex )

Definition at line 250 of file ImportPipelineHelpers.cpp.

Here is the call graph for this function:

◆ findAmountTokenIndices()

std::vector< size_t > core::application::importing::internal::findAmountTokenIndices ( const core::application::importing::transaction::internal::OcrLine & line,
int valutaX = -1,
int bandPx = 0 )
noexcept

Finds the indices of amount-like tokens in an OCR line.

Parameters
lineOCR line to inspect.
valutaXOptional currency column reference.
bandPxMatching band in pixels around the reference column.
Returns
Indices of tokens that look like amounts.

Definition at line 38 of file ParserHelpers.cpp.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ findAndParseAmountInLine()

std::optional< double > core::application::importing::internal::findAndParseAmountInLine ( const core::application::importing::transaction::internal::OcrLine & line,
int valutaX = -1,
std::vector< std::string > * debugOut = nullptr )
noexcept

Finds and parses an amount token inside an OCR line.

Parameters
lineOCR line to inspect.
valutaXReference currency column.
debugOutOptional debug output sink.
Returns
Parsed amount when a valid token is found.

Definition at line 230 of file ParserHelpers.cpp.

Here is the call graph for this function:

◆ findFirstFullDate()

std::optional< std::string > core::application::importing::internal::findFirstFullDate ( const std::string & text)
noexcept

Finds the first full date in a text fragment.

Parameters
textInput text to inspect.
Returns
First detected full date, or an empty optional when none is present.

Definition at line 34 of file ParserDateUtils.cpp.

◆ findPhraseCenterX()

std::optional< int > core::application::importing::internal::findPhraseCenterX ( const core::application::importing::transaction::internal::OcrLine & line,
std::initializer_list< const char * > phraseLower )
noexcept

Locates the horizontal center of a multi-token phrase in an OCR line.

Parameters
lineOCR line to inspect.
phraseLowerLower-case phrase tokens to find.
Returns
Phrase center x-coordinate when the phrase is present.

Definition at line 100 of file ParserHelpers.cpp.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ findTokenCenterX()

std::optional< int > core::application::importing::internal::findTokenCenterX ( const core::application::importing::transaction::internal::OcrLine & line,
const std::string & tokenLower )
noexcept

Locates the horizontal center of a token inside an OCR line.

Parameters
lineOCR line to inspect.
tokenLowerLower-case token to find.
Returns
Token center x-coordinate when the token is present.

Definition at line 74 of file ParserHelpers.cpp.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ groupMergeLinesRaw()

std::vector< RawLineLite > core::application::importing::internal::groupMergeLinesRaw ( const std::vector< RawLineLite > & lines,
int maxGapPx )

Merges raw lines using a gap-based heuristic.

Parameters
linesRaw lines to merge.
maxGapPxMaximum gap between lines that should still be merged.
Returns
Merged raw lines.

Definition at line 527 of file ParserHelpers.cpp.

Here is the caller graph for this function:

◆ hasAmountLikeTokenInLine()

bool core::application::importing::internal::hasAmountLikeTokenInLine ( const core::application::importing::transaction::internal::OcrLine & line,
int valutaX )
noexcept

Checks whether the line contains amount-like tokens near the valuta column.

Parameters
lineOCR line to inspect.
valutaXReference currency column.
Returns
True when amount-like content is found near the valuta column.

Definition at line 149 of file ParserHelpers.cpp.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ hasAmountNearValuta()

bool core::application::importing::internal::hasAmountNearValuta ( const core::application::importing::transaction::internal::OcrLine & line,
int valutaX,
int bandPx )
noexcept

Checks whether an amount appears close to the valuta reference.

Parameters
lineOCR line to inspect.
valutaXReference currency column.
bandPxAllowed band around the reference x-coordinate.
Returns
True when an amount-like token is near the currency reference.

Definition at line 184 of file ParserHelpers.cpp.

Here is the call graph for this function:

◆ hasLeftDescriptiveText()

bool core::application::importing::internal::hasLeftDescriptiveText ( const core::application::importing::transaction::internal::OcrLine & line,
int valutaX )
noexcept

Checks whether the line contains descriptive text to the left of the valuta column.

Parameters
lineOCR line to inspect.
valutaXReference currency column.
Returns
True when descriptive text is detected on the left side.

Definition at line 162 of file ParserHelpers.cpp.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ hasShortDateToken()

bool core::application::importing::internal::hasShortDateToken ( const std::string & text)
noexcept

Checks whether the text contains a short date token.

Parameters
textInput text to inspect.
Returns
True when a short date token is present.

Definition at line 20 of file ParserDateUtils.cpp.

Here is the caller graph for this function:

◆ hasTokenNearX()

bool core::application::importing::internal::hasTokenNearX ( const core::application::importing::transaction::internal::OcrLine & line,
int x,
int bandPx )
noexcept

Checks whether any token in the line is close to a horizontal position.

Parameters
lineOCR line to inspect.
xReference x-coordinate.
bandPxAllowed band around the reference x-coordinate.
Returns
True when a token lies inside the band.

Definition at line 132 of file ParserHelpers.cpp.

Here is the call graph for this function:

◆ inferColumnModelFromLines() [1/2]

ColumnGuess core::application::importing::internal::inferColumnModelFromLines ( const std::vector< core::application::importing::transaction::internal::OcrLine > & lines,
size_t scanN = 16 )
noexcept

Infers a column layout from a set of OCR lines.

Parameters
linesOCR lines to scan.
scanNMaximum number of leading lines to inspect.
Returns
Inferred column guess.
Here is the call graph for this function:

◆ inferColumnModelFromLines() [2/2]

ColumnGuess core::application::importing::internal::inferColumnModelFromLines ( const std::vector< OcrLine > & lines,
size_t scanN )
noexcept

Definition at line 497 of file ParserHelpers.cpp.

Here is the call graph for this function:

◆ isDebitCreditHeaderLine()

bool core::application::importing::internal::isDebitCreditHeaderLine ( const std::string & line)

Detects whether a line looks like a debit/credit table header.

Parameters
lineInput line to inspect.
Returns
True when the line resembles a debit/credit header row.

Definition at line 155 of file ParserHeuristics.cpp.

◆ isFooterLine()

bool core::application::importing::internal::isFooterLine ( const std::string & line)

Detects whether a line marks a footer section.

Parameters
lineInput line to inspect.
Returns
True when the line is treated as a footer.

Definition at line 106 of file ParserHeuristics.cpp.

Here is the call graph for this function:

◆ isHeaderNoiseLine()

bool core::application::importing::internal::isHeaderNoiseLine ( const std::string & line)

Detects whether a line looks like header noise.

Parameters
lineInput line to inspect.
Returns
True when the line should be treated as header noise.

Definition at line 63 of file ParserHeuristics.cpp.

Here is the call graph for this function:

◆ isLooseTransactionLine()

bool core::application::importing::internal::isLooseTransactionLine ( const core::application::importing::transaction::internal::OcrLine & line,
int valutaX )
noexcept

Checks whether the line looks like a loosely formatted transaction row.

Parameters
lineOCR line to inspect.
valutaXReference currency column.
Returns
True when the line resembles a transaction row.

Definition at line 195 of file ParserHelpers.cpp.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ isPostTransactionFootnote()

bool core::application::importing::internal::isPostTransactionFootnote ( const std::string & line)

Detects whether a line belongs to the post-transaction footnote area.

Parameters
lineInput line to inspect.
Returns
True when the line should be treated as a post-transaction footnote.

Definition at line 112 of file ParserHeuristics.cpp.

Here is the call graph for this function:

◆ isTransactionsSectionHeader()

bool core::application::importing::internal::isTransactionsSectionHeader ( const std::string & line)

Detects whether a line marks the start of the transactions section.

Parameters
lineInput line to inspect.
Returns
True when the line is considered a transactions-section header.

Definition at line 144 of file ParserHeuristics.cpp.

Here is the call graph for this function:

◆ normalizeAlnumLower()

std::string core::application::importing::internal::normalizeAlnumLower ( const std::string & s)
noexcept

Normalizes alphanumeric text to lower-case ASCII for matching.

Parameters
sInput string.
Returns
Normalized string.

Definition at line 64 of file ParserHelpers.cpp.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ processImportPage() [1/2]

PageWork core::application::importing::internal::processImportPage ( size_t pageIndex,
size_t totalPages,
const ImportRequest & req,
const core::ports::pdf_rendering::RenderResult & renderRes,
const core::ports::pdf_rendering::ExtractResult & extractRes,
const std::shared_ptr< core::ports::document_image_processing::IDocumentImageProcessor > & documentImageProcessor,
const std::shared_ptr< core::ports::text_recognition::ITextRecognizer > & tesseract,
core::jobs::SlotLimiter * ocrLimiter,
const ProgressReporter & report,
std::atomic< size_t > & doneUnits,
size_t totalUnits,
core::ports::diagnostics::IErrorReporter * errorReporter,
ImportResult & out,
std::mutex & artifactsMutex )

◆ processImportPage() [2/2]

PageWork core::application::importing::internal::processImportPage ( size_t pageIndex,
size_t totalPages,
const ImportRequest & req,
const poppler::RenderResult & renderRes,
const poppler::ExtractResult & extractRes,
const std::shared_ptr< core::ports::document_image_processing::IDocumentImageProcessor > & documentImageProcessor,
const std::shared_ptr< core::ports::text_recognition::ITextRecognizer > & tesseract,
core::jobs::SlotLimiter * ocrLimiter,
const ProgressReporter & report,
std::atomic< size_t > & doneUnits,
size_t totalUnits,
core::ports::diagnostics::IErrorReporter * errorReporter,
ImportResult & out,
std::mutex & artifactsMutex )

Definition at line 112 of file ImportPipelineHelpers.cpp.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ readImportBytes() [1/2]

std::vector< uint8_t > core::application::importing::internal::readImportBytes ( const std::filesystem::path & path)

Definition at line 96 of file ImportPipelineHelpers.cpp.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ readImportBytes() [2/2]

std::vector< uint8_t > core::application::importing::internal::readImportBytes ( const std::filesystem::path & path,
core::ports::diagnostics::IErrorReporter * errorReporter )

Definition at line 101 of file ImportPipelineHelpers.cpp.

Here is the call graph for this function:

◆ splitMainRowFromOcrLine()

core::application::importing::transaction::internal::TransactionMainRow core::application::importing::internal::splitMainRowFromOcrLine ( const core::application::importing::transaction::internal::OcrLine & src,
int valutaX,
int debitX,
int creditX )
noexcept

Splits an OCR line into the main transaction row representation.

Parameters
srcOCR line input.
valutaXReference currency column.
debitXReference debit column.
creditXReference credit column.
Returns
Main transaction row extracted from the OCR line.

Definition at line 697 of file ParserHelpers.cpp.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ splitMainRowFromRaw()

core::application::importing::transaction::internal::TransactionMainRow core::application::importing::internal::splitMainRowFromRaw ( const RawLineLite & src,
int valutaX,
int debitX,
int creditX )
noexcept

Splits a raw line into the main transaction row representation.

Parameters
srcRaw line container.
valutaXReference currency column.
debitXReference debit column.
creditXReference credit column.
Returns
Main transaction row extracted from the raw line.

Definition at line 578 of file ParserHelpers.cpp.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ toOcrLineFromRawWords()

core::application::importing::transaction::internal::OcrLine core::application::importing::internal::toOcrLineFromRawWords ( const RawLineLite & src,
size_t i0,
size_t i1 )
noexcept

Converts a range of raw words into one OCR line.

Parameters
srcRaw line container.
i0Start word index.
i1End word index.
Returns
OCR line built from the selected words.

Definition at line 550 of file ParserHelpers.cpp.

Here is the call graph for this function:
Here is the caller graph for this function:

Variable Documentation

◆ parserConfig

ParserConfig core::application::importing::internal::parserConfig

Shared parser configuration instance used by import parsing helpers.

Definition at line 12 of file ParserHelpers.cpp.