leca/checks-parser
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 3 Wiki Activity

more advanced parsing

Browse Source
This commit is contained in:
leca
2025-06-10 22:43:44 +03:00
parent 9da589839c
commit c9f447009b
9 changed files with 149 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
email_parser/emailparser.cpp
View File

@@ -1,5 +1,3 @@
#include "utils/utils.h"
#include "utils/base64.h"
#include <email_parser/emailparser.h>
#include <opencv2/opencv.hpp>
@@ -9,6 +7,7 @@
#include <opencv2/videoio.hpp>
#include <utils/utils.h>
#include <utils/base64.h>
#include <check/check.h>
#include <boost/regex.hpp>
#include <boost/algorithm/string/regex.hpp>
@@ -37,28 +36,21 @@ std::map<std::string, std::string> EmailParser::parse(std::string &email_content
//3. Search "t=\d{8}T\d{4,6}&s=\d{1,6}\.\d{1,2}&fn=\d{10,16}&i=\d{6}&fp=\d{10}&n=\d". Note that in some emails = and & signs could be replaced with its code in HTTP requests: %3D, %26
// 3.1 If not found, notify the user that we could not parse the .eml file
/* Find content-types */
Check c;
std::vector<std::pair<int, int>> content_types = {};
boost::regex content_type_regex("Content-Type: image/(gif|png|jpg)");
/* Find image content-types */
boost::regex images_content_type_regex("Content-Type: image/(gif|png|jpg)");
boost::regex text_content_types_regex("Content-Type: text\\/(html|plain)");
boost::regex part_end_regex("--.{5,48}");
// boost::regex check_data_content("t=\\d+T\\d+&s=\\d+\\.\\d+&fn=\\d{16}&i=\\d{4,5}&fp=\\d{10}&n=\\d");
boost::regex to_erase("([\\w-]+:\\s*.{2,64}\\r\\n)+");
boost::regex to_erase_two("--.{5,48}");
for (boost::sregex_iterator it{email_content.begin(), email_content.end(), content_type_regex}, end{}; it != end; it ++) {
unsigned int start_position = it->position(), end_position = -1;
std::vector<std::pair<int, int>> images_content_parts = find_parts(images_content_type_regex, part_end_regex, email_content);
for (boost::sregex_iterator it2{email_content.begin() + start_position, email_content.end(), part_end_regex}, end2{}; it2 != end2; it2++) {
end_position = it2->position();
break;
}
content_types.push_back(std::pair<int, int>(start_position, end_position));
}
/* iterate through found image content-types and try searching qr codes, decode them and see if it's the needed data */
/* iterate through found content-types and try searching qr codes, decode them and see if it's the needed data */
for (unsigned int i = 0; i < images_content_parts.size(); i ++) {
for (unsigned int i = 0; i < content_types.size(); i ++) {
boost::regex to_erase("(Content.{5,64}\\r\\n)+");
boost::regex to_erase_two("--.{5,48}");
std::string part = email_content.substr(content_types[i].first, content_types[i].second);
std::string part = email_content.substr(images_content_parts[i].first, images_content_parts[i].second);
boost::erase_regex(part, to_erase);
boost::erase_regex(part, to_erase_two);
part.erase(std::remove(part.begin(), part.end(), '\r'), part.end());
@@ -80,10 +72,11 @@ std::map<std::string, std::string> EmailParser::parse(std::string &email_content
}
cv::QRCodeDetector qrDecoder = cv::QRCodeDetector();
std::string decoded_qr_params = qrDecoder.detectAndDecode(image);
boost::regex check_data_content("t=\\d+T\\d+&s=\\d+\\.\\d+&fn=\\d{16}&i=\\d{4,5}&fp=\\d{10}&n=\\d");
if (boost::regex_match(decoded_qr_params, check_data_content)) {
std::map<std::string, std::string> paramsMap = get_params_from_string(decoded_qr_params);
std::string decoded_qr = qrDecoder.detectAndDecode(image);
std::string parameters = find_check_parameters(decoded_qr);
if (parameters != "") {
std::map<std::string, std::string> paramsMap = get_params_from_string(parameters);
return paramsMap;
}
@@ -91,17 +84,105 @@ std::map<std::string, std::string> EmailParser::parse(std::string &email_content
/* If the E-Mail has no QR code in it as a separate part, there's posibilly a QR code inserted using html's tag <img> with base64-encoded image. Try searching it */
std::vector<std::pair<int, int>> texts_content_parts = find_parts(text_content_types_regex, part_end_regex, email_content);
for (unsigned int i = 0; i < texts_content_parts.size(); i ++) {
std::string part = email_content.substr(texts_content_parts[i].first, texts_content_parts[i].second);
boost::erase_regex(part, to_erase);
boost::erase_regex(part, to_erase_two);
//If there's '<' character, most likely that the part's content is plain html, otherwise it's most likely a base64 encoded html.
if (part.find("<") == std::string::npos) {
part.erase(std::remove(part.begin(), part.end(), '\r'), part.end());
part.erase(std::remove(part.begin(), part.end(), '\n'), part.end());
part = base64_decode(part);
}
std::string parameters = find_check_parameters(part);
if (parameters != "") {
std::map<std::string, std::string> paramsMap = get_params_from_string(parameters);
return paramsMap;
}
std::string url = extract_qr_url_from_img(part);
Net n;
std::string path = get_path_relative_to_home(".local/share/checks_parser/tmp");
n.get_file(url, path);
std::ifstream ifile(path, std::ios::in | std::ios::binary);
const unsigned int size = std::filesystem::file_size(path);
std::string qr_code_contents(size, '\0');
ifile.read(qr_code_contents.data(), size);
std::vector<uchar> data(qr_code_contents.begin(), qr_code_contents.end());
cv::Mat image = cv::imdecode(cv::Mat(data), 1);
cv::QRCodeDetector qrDecoder = cv::QRCodeDetector();
std::string decoded_qr = qrDecoder.detectAndDecode(image);
parameters = find_check_parameters(decoded_qr);
if (parameters != "") {
std::map<std::string, std::string> paramsMap = get_params_from_string(parameters);
return paramsMap;
}
}
/* If there's no such case, the last chance is <img src="..."> which will have a link with needed parameters or the qr code that should be downloaded and decoded */
/* If the code has reached this part and found nothing, it's most likely that there are no QR codes at all. */
// return Check();
return std::map<std::string, std::string>();
}
std::map<std::string, std::string> EmailParser::parse_file(std::string path) {
std::ifstream ifile(path, std::ios::in | std::ios::binary);
const unsigned int size = std::filesystem::file_size(path);
std::string content(size, '\0');
ifile.read(content.data(), size);
// std::cout << "Parsing " << path << std::endl;
// std::ifstream ifile(path, std::ios::in | std::ios::binary);
// const unsigned int size = std::filesystem::file_size(path);
// std::string content(size, '\0');
// ifile.read(content.data(), size);
std::string content = read_file(path);
return parse(content);
return std::map<std::string, std::string>();
}
std::vector<std::pair<int, int>> EmailParser::find_parts(const boost::regex &start_regex, const boost::regex &end_regex, const std::string &content) {
std::vector<std::pair<int, int>> parts = {};
for (boost::sregex_iterator it{content.begin(), content.end(), start_regex}, end{}; it != end; it ++) {
unsigned int start_position = it->position(), end_position = content.length();
for (boost::sregex_iterator it2{content.begin() + start_position, content.end(), end_regex}, end2{}; it2 != end2; it2++) {
end_position = it2->position();
break;
}
parts.push_back(std::pair<int, int>(start_position, end_position));
}
return parts;
}
std::string EmailParser::find_check_parameters(std::string &part) {
boost::regex params_regex ("t(=|%3d)\\d+T\\d+(&|%26)s\\1\\d+\\.\\d+\\2fn\\1\\d{16}\\2i\\1\\d{3,6}\\2fp\\1\\d{9,10}\\2n\\1\\d", boost::regex::icase);
for (boost::sregex_iterator it{part.begin(), part.end(), params_regex}, end{}; it != end; it++) {
return it->str();
}
return "";
}
std::string EmailParser::extract_qr_url_from_img(std::string &part) {
std::string url = "";
boost::regex img_tag_regex("<img[^\\n\\r<]*>");
boost::regex img_url_str("https?:\\/\\/.*(qr(code)?)[^\\n\\r\"]+", boost::regex::icase);
for (boost::sregex_iterator it{part.begin(), part.end(), img_tag_regex}, end{}; it != end; it++) {
std::string img_tag = it->str();
for (boost::sregex_iterator it2{img_tag.begin(), img_tag.end(), img_url_str}, end2{}; it2 != end2; it2++) {
return it2->str();
}
}
return url;
}

5
email_parser/emailparser.h
View File

@@ -3,13 +3,16 @@
#include <check/check.h>
#include <map>
#include <boost/regex.hpp>
class EmailParser {
public:
EmailParser();
std::map<std::string, std::string> parse(std::string &email_content);
std::map<std::string, std::string> parse_file(std::string path);
std::vector<std::pair<int, int>> find_parts(const boost::regex &start_regex, const boost::regex &end_regex, const std::string &content);
std::string find_check_parameters(std::string &part);
std::string extract_qr_url_from_img(std::string &part);
};
#endif // CHECKS_PARSER_EMAIL_PARSER