From 7b587d82c874df275f147eb461f6389163547e4e Mon Sep 17 00:00:00 2001 From: frosty Date: Mon, 3 Aug 2026 23:21:41 -0400 Subject: feat: add yacy support --- src/Scraping/Scraping.c | 36 +++++++++---- src/Scraping/Scraping.h | 2 + src/Scraping/ScrapingHttp.c | 13 +++-- src/Scraping/ScrapingParsers.c | 116 +++++++++++++++++++++++++++++++++++++---- 4 files changed, 142 insertions(+), 25 deletions(-) (limited to 'src/Scraping') diff --git a/src/Scraping/Scraping.c b/src/Scraping/Scraping.c index b81c216..5c51ca5 100644 --- a/src/Scraping/Scraping.c +++ b/src/Scraping/Scraping.c @@ -4,6 +4,7 @@ #include "Config.h" #include #include +#include #include #include #include @@ -24,7 +25,8 @@ static int response_is_startpage_captcha(const ScrapeJob *job, return response_contains(response, "Startpage Captcha") || response_contains(response, "Startpage Captcha") || - response_contains(response, "/static-pages-assets/page-data/captcha/") || + response_contains(response, + "/static-pages-assets/page-data/captcha/") || response_contains(response, ">Startpage Blocked"); } @@ -39,7 +41,8 @@ static int response_looks_like_results_page(const ScrapeJob *job, } if (strcmp(job->engine->name, "Startpage") == 0) { - return response_contains(response, "Startpage Search Results") || + return response_contains(response, + "Startpage Search Results") || response_contains(response, "class=\"w-gl") || response_contains(response, "data-testid=\"gl-title-link\""); } @@ -55,6 +58,11 @@ static int response_looks_like_results_page(const ScrapeJob *job, response_contains(response, "Mojeek Search"); } + if (strcmp(job->engine->name, "YaCy") == 0) { + return response_contains(response, "engine->is_xml) { + doc = xmlReadMemory(response, response_size, NULL, NULL, + XML_PARSE_RECOVER | XML_PARSE_NOERROR | + XML_PARSE_NOWARNING); + } else { + doc = htmlReadMemory(response, response_size, NULL, NULL, + HTML_PARSE_RECOVER | HTML_PARSE_NOERROR | + HTML_PARSE_NOWARNING); + } if (!doc) { job->status = SCRAPE_STATUS_FETCH_ERROR; return; } - job->results_count = - job->engine->parser(job->engine->name, doc, job->out_results, - job->max_results); + job->results_count = job->engine->parser(job->engine->name, doc, + job->out_results, job->max_results); xmlFreeDoc(doc); if (job->results_count > 0) { @@ -200,7 +214,8 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) { return -1; } - for (char *p = encoded_query + strlen(encoded_query) - 3; p >= encoded_query; p--) { + for (char *p = encoded_query + strlen(encoded_query) - 3; p >= encoded_query; + p--) { if (p[0] == '%' && p[1] == '2' && p[2] == '0') { *p = '+'; memmove(p + 1, p + 3, strlen(p + 3) + 1); @@ -279,8 +294,7 @@ int should_retry(ScrapeJob *jobs, int num_jobs) { int scrape_engines_parallel(ScrapeJob *jobs, int num_jobs) { int retries = 0; -retry: - ; +retry:; CURLM *multi_handle = curl_multi_init(); if (!multi_handle) return -1; diff --git a/src/Scraping/Scraping.h b/src/Scraping/Scraping.h index be65e5a..14e6e55 100644 --- a/src/Scraping/Scraping.h +++ b/src/Scraping/Scraping.h @@ -26,6 +26,7 @@ typedef struct { int page_base; ParserFunc parser; int enabled; + int is_xml; } SearchEngine; typedef struct { @@ -59,6 +60,7 @@ typedef struct { extern SearchEngine ENGINE_REGISTRY[]; extern const int ENGINE_COUNT; void apply_engines_config(const char *engines_str); +void configure_yacy_engine(const char *instance); size_t write_memory_callback(void *contents, size_t size, size_t nmemb, void *userp); diff --git a/src/Scraping/ScrapingHttp.c b/src/Scraping/ScrapingHttp.c index 1a6a292..89f4853 100644 --- a/src/Scraping/ScrapingHttp.c +++ b/src/Scraping/ScrapingHttp.c @@ -89,11 +89,14 @@ struct curl_slist *build_request_headers(const char *host_header, struct curl_slist *headers = NULL; char host_buf[BUFFER_SIZE_MEDIUM], ref_buf[BUFFER_SIZE_MEDIUM]; - snprintf(host_buf, sizeof(host_buf), "Host: %s", host_header); - snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer); - - headers = curl_slist_append(headers, host_buf); - headers = curl_slist_append(headers, ref_buf); + if (host_header && host_header[0] != '\0') { + snprintf(host_buf, sizeof(host_buf), "Host: %s", host_header); + headers = curl_slist_append(headers, host_buf); + } + if (referer && referer[0] != '\0') { + snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer); + headers = curl_slist_append(headers, ref_buf); + } headers = curl_slist_append( headers, "Accept: " diff --git a/src/Scraping/ScrapingParsers.c b/src/Scraping/ScrapingParsers.c index 96aaded..2e5689b 100644 --- a/src/Scraping/ScrapingParsers.c +++ b/src/Scraping/ScrapingParsers.c @@ -249,8 +249,8 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc, if (!ctx) return 0; - xmlXPathObjectPtr obj = - xml_xpath_eval(ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]"); + xmlXPathObjectPtr obj = xml_xpath_eval( + ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]"); if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) { free_xpath_objects(ctx, obj); @@ -268,18 +268,17 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc, xmlNodePtr result_node = obj->nodesetval->nodeTab[i]; ctx->node = result_node; - xmlXPathObjectPtr link_obj = - xml_xpath_eval(ctx, ".//a[@class='title']"); + xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, ".//a[@class='title']"); char *url = (link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0) ? (char *)xmlGetProp(link_obj->nodesetval->nodeTab[0], (xmlChar *)"href") : NULL; - char *title = (link_obj && link_obj->nodesetval && - link_obj->nodesetval->nodeNr > 0) - ? xml_node_content(link_obj->nodesetval->nodeTab[0]) - : NULL; + char *title = + (link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0) + ? xml_node_content(link_obj->nodesetval->nodeTab[0]) + : NULL; xmlXPathObjectPtr snippet_obj = xml_xpath_eval(ctx, ".//p[@class='s']"); char *snippet_text = @@ -310,6 +309,90 @@ static int parse_yahoo(const char *engine_name, xmlDocPtr doc, static int parse_mojeek(const char *engine_name, xmlDocPtr doc, SearchResult **out_results, int max_results); +static int parse_yacy(const char *engine_name, xmlDocPtr doc, + SearchResult **out_results, int max_results) { + (void)engine_name; + int found_count = 0; + + xmlXPathContextPtr ctx = create_xpath_context(doc); + if (!ctx) + return 0; + + xmlXPathObjectPtr obj = xml_xpath_eval(ctx, "//item"); + + if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) { + free_xpath_objects(ctx, obj); + return 0; + } + + int num_items = obj->nodesetval->nodeNr; + *out_results = alloc_results_array(num_items, max_results); + if (!*out_results) { + free_xpath_objects(ctx, obj); + return 0; + } + + for (int i = 0; i < num_items && found_count < max_results; i++) { + xmlNodePtr item_node = obj->nodesetval->nodeTab[i]; + ctx->node = item_node; + + xmlXPathObjectPtr title_obj = xml_xpath_eval(ctx, "./title"); + char *title = (title_obj && title_obj->nodesetval && + title_obj->nodesetval->nodeNr > 0) + ? xml_node_content(title_obj->nodesetval->nodeTab[0]) + : NULL; + + xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, "./link"); + char *url = + (link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0) + ? xml_node_content(link_obj->nodesetval->nodeTab[0]) + : NULL; + + xmlXPathObjectPtr desc_obj = xml_xpath_eval(ctx, "./description"); + char *snippet_text = + (desc_obj && desc_obj->nodesetval && desc_obj->nodesetval->nodeNr > 0) + ? xml_node_content(desc_obj->nodesetval->nodeTab[0]) + : NULL; + + if (url && title) { + assign_result(&(*out_results)[found_count], url, title, snippet_text, 0); + found_count++; + } + + free_xml_node_list(title, url, snippet_text); + if (title_obj) + xmlXPathFreeObject(title_obj); + if (link_obj) + xmlXPathFreeObject(link_obj); + if (desc_obj) + xmlXPathFreeObject(desc_obj); + } + + ctx->node = NULL; + free_xpath_objects(ctx, obj); + return found_count; +} + +static char yacy_base_url[BUFFER_SIZE_LARGE]; + +void configure_yacy_engine(const char *instance) { + if (!instance || instance[0] == '\0') + instance = DEFAULT_YACY_INSTANCE; + + snprintf(yacy_base_url, sizeof(yacy_base_url), + "%s/" + "yacysearch.rss?resource=global&urlmaskfilter=.*&prefermaskfilter=&" + "nav=all&maximumRecords=%d&query=", + instance, MAX_RESULTS_PER_ENGINE); + + for (int i = 0; i < ENGINE_COUNT; i++) { + if (strcmp(ENGINE_REGISTRY[i].id, "yacy") == 0) { + ENGINE_REGISTRY[i].base_url = yacy_base_url; + break; + } + } +} + SearchEngine ENGINE_REGISTRY[] = { {.id = "ddg", .name = "DuckDuckGo Lite", @@ -350,7 +433,18 @@ SearchEngine ENGINE_REGISTRY[] = { .page_multiplier = 10, .page_base = 1, .parser = parse_mojeek, - .enabled = 1}}; + .enabled = 1}, + {.id = "yacy", + .name = "YaCy", + .base_url = "", + .host_header = NULL, + .referer = NULL, + .page_param = "startRecord", + .page_multiplier = 10, + .page_base = 0, + .parser = parse_yacy, + .enabled = 0, + .is_xml = 1}}; const int ENGINE_COUNT = sizeof(ENGINE_REGISTRY) / sizeof(SearchEngine); @@ -373,6 +467,8 @@ static int engine_id_compare(const char *engine_id, const char *config_id) { void apply_engines_config(const char *engines_str) { if (!engines_str || engines_str[0] == '\0') { for (int i = 0; i < ENGINE_COUNT; i++) { + if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy")) + continue; ENGINE_REGISTRY[i].enabled = 1; } return; @@ -395,6 +491,8 @@ void apply_engines_config(const char *engines_str) { if (strcmp(token, "*") == 0) { for (int i = 0; i < ENGINE_COUNT; i++) { + if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy")) + continue; ENGINE_REGISTRY[i].enabled = 1; } } else if (token[0] == '-' && token[1] != '\0') { -- cgit v1.3 From 10d44e7ff547a806e4bf109e7aadd48488ddc9f0 Mon Sep 17 00:00:00 2001 From: frosty Date: Wed, 12 Aug 2026 15:40:41 -0400 Subject: fix: improvements allocation safety and string processing --- src/Config.c | 66 +++++++++++++++---------------------- src/Routes/Search.c | 77 ++++++++++++++++++++----------------------- src/Scraping/Scraping.c | 14 +++++--- src/Utility/Utility.c | 88 +++++++++---------------------------------------- src/Utility/Utility.h | 8 ----- 5 files changed, 87 insertions(+), 166 deletions(-) (limited to 'src/Scraping') diff --git a/src/Config.c b/src/Config.c index bde76fe..3d2790b 100644 --- a/src/Config.c +++ b/src/Config.c @@ -3,6 +3,17 @@ #include #include +static char *trim(char *value) { + while (*value == ' ' || *value == '\t') + value++; + + char *end = value + strlen(value); + while (end > value && (end[-1] == ' ' || end[-1] == '\t')) + end--; + *end = '\0'; + return value; +} + int load_config(const char *filename, Config *config) { FILE *file = fopen(filename, "r"); if (!file) { @@ -33,51 +44,31 @@ int load_config(const char *filename, Config *config) { char *delimiter = strchr(line, '='); if (delimiter) { *delimiter = '\0'; - char *key = line; - char *value = delimiter + 1; - - while (*key == ' ' || *key == '\t') - key++; - while (*value == ' ' || *value == '\t') - value++; + char *key = trim(line); + char *value = trim(delimiter + 1); - char *key_end = key + strlen(key) - 1; - while (key_end > key && (*key_end == ' ' || *key_end == '\t')) { - *key_end = '\0'; - key_end--; - } - - char *value_end = value + strlen(value) - 1; - while (value_end > value && (*value_end == ' ' || *value_end == '\t' || - *value_end == '"' || *value_end == '\'')) { - *value_end = '\0'; - value_end--; - } - - while (*value == ' ' || *value == '\t') - value++; + char *value_end = value + strlen(value); + while (value_end > value && + (value_end[-1] == '"' || value_end[-1] == '\'')) + *--value_end = '\0'; while (*value == '"' || *value == '\'') value++; if (strcmp(section, "server") == 0) { if (strcmp(key, "host") == 0) { - strncpy(config->host, value, sizeof(config->host) - 1); - config->host[sizeof(config->host) - 1] = '\0'; + snprintf(config->host, sizeof(config->host), "%s", value); } else if (strcmp(key, "port") == 0) { config->port = atoi(value); } else if (strcmp(key, "locale") == 0) { - strncpy(config->default_locale, value, - sizeof(config->default_locale) - 1); - config->default_locale[sizeof(config->default_locale) - 1] = '\0'; + snprintf(config->default_locale, sizeof(config->default_locale), "%s", + value); } } else if (strcmp(section, "proxy") == 0) { if (strcmp(key, "proxy") == 0) { - strncpy(config->proxy, value, sizeof(config->proxy) - 1); - config->proxy[sizeof(config->proxy) - 1] = '\0'; + snprintf(config->proxy, sizeof(config->proxy), "%s", value); } else if (strcmp(key, "list_file") == 0) { - strncpy(config->proxy_list_file, value, - sizeof(config->proxy_list_file) - 1); - config->proxy_list_file[sizeof(config->proxy_list_file) - 1] = '\0'; + snprintf(config->proxy_list_file, sizeof(config->proxy_list_file), + "%s", value); } else if (strcmp(key, "max_retries") == 0) { config->max_proxy_retries = atoi(value); } else if (strcmp(key, "randomize_username") == 0) { @@ -87,8 +78,7 @@ int load_config(const char *filename, Config *config) { } } else if (strcmp(section, "cache") == 0) { if (strcmp(key, "dir") == 0) { - strncpy(config->cache_dir, value, sizeof(config->cache_dir) - 1); - config->cache_dir[sizeof(config->cache_dir) - 1] = '\0'; + snprintf(config->cache_dir, sizeof(config->cache_dir), "%s", value); } else if (strcmp(key, "ttl_search") == 0) { config->cache_ttl_search = atoi(value); } else if (strcmp(key, "ttl_infobox") == 0) { @@ -98,12 +88,10 @@ int load_config(const char *filename, Config *config) { } } else if (strcmp(section, "engines") == 0) { if (strcmp(key, "engines") == 0) { - strncpy(config->engines, value, sizeof(config->engines) - 1); - config->engines[sizeof(config->engines) - 1] = '\0'; + snprintf(config->engines, sizeof(config->engines), "%s", value); } else if (strcmp(key, "yacy_instance") == 0) { - strncpy(config->yacy_instance, value, - sizeof(config->yacy_instance) - 1); - config->yacy_instance[sizeof(config->yacy_instance) - 1] = '\0'; + snprintf(config->yacy_instance, sizeof(config->yacy_instance), "%s", + value); } } else if (strcmp(section, "rate_limit") == 0) { if (strcmp(key, "search_requests") == 0) { diff --git a/src/Routes/Search.c b/src/Routes/Search.c index abc2e73..a245f6b 100644 --- a/src/Routes/Search.c +++ b/src/Routes/Search.c @@ -251,22 +251,20 @@ static void *infobox_thread_func(void *arg) { return NULL; } -static int add_infobox_to_collection(InfoBox *infobox, char ****collection, - int **inner_counts, int current_count) { +static int add_infobox_to_collection(InfoBox *infobox, + StringMatrix *collection) { const char *values[INFOBOX_FIELD_COUNT] = { infobox->title, infobox->thumbnail_url, infobox->extract, infobox->url, infobox->url, }; - return append_string_row(collection, inner_counts, current_count, values, - INFOBOX_FIELD_COUNT); + return string_matrix_append(collection, values, INFOBOX_FIELD_COUNT); } static int add_warning_to_collection(const char *engine_name, const char *warning_message, - char ****collection, int **inner_counts, - int current_count) { + StringMatrix *collection) { const char *values[] = {engine_name, warning_message}; - return append_string_row(collection, inner_counts, current_count, values, 2); + return string_matrix_append(collection, values, 2); } static void free_user_engine_list(char **user_engines, int user_engine_count) { @@ -587,15 +585,14 @@ int results_handler(UrlParams *params) { } if (filter_engine_count > 1) { - char ***filter_matrix = NULL; - int *filter_inner_counts = NULL; - int filter_count = 0; + StringMatrix filter_matrix; + string_matrix_init(&filter_matrix); char *all_href = build_search_href(raw_query, "all", 1); - - filter_count = add_link_to_collection( + const char *all_values[LINK_FIELD_COUNT] = { all_href, "All", - selected_engine ? "engine-filter" : "engine-filter active", - &filter_matrix, &filter_inner_counts, filter_count); + selected_engine ? "engine-filter" : "engine-filter active"}; + + string_matrix_append(&filter_matrix, all_values, LINK_FIELD_COUNT); free(all_href); for (int i = 0; i < ENGINE_COUNT; i++) { @@ -611,17 +608,18 @@ int results_handler(UrlParams *params) { ? "engine-filter active" : "engine-filter"; - filter_count = add_link_to_collection( - filter_href, ENGINE_REGISTRY[i].name, filter_class, &filter_matrix, - &filter_inner_counts, filter_count); + const char *filter_values[LINK_FIELD_COUNT] = { + filter_href, ENGINE_REGISTRY[i].name, filter_class}; + string_matrix_append(&filter_matrix, filter_values, LINK_FIELD_COUNT); free(filter_href); } - if (filter_count > 0) { - context_set_array_of_arrays(&ctx, "engine_filters", filter_matrix, - filter_count, filter_inner_counts); - free_string_matrix(filter_matrix, filter_inner_counts, filter_count); + if (filter_matrix.count > 0) { + context_set_array_of_arrays(&ctx, "engine_filters", filter_matrix.rows, + filter_matrix.count, + filter_matrix.field_counts); } + string_matrix_free(&filter_matrix); } if (page == 1) { @@ -673,25 +671,23 @@ int results_handler(UrlParams *params) { return 0; } - char ***infobox_matrix = NULL; - int *infobox_inner_counts = NULL; - int infobox_count = 0; + StringMatrix infobox_matrix; + string_matrix_init(&infobox_matrix); if (page == 1) { for (int i = 0; i < HANDLER_COUNT; i++) { if (infobox_data[i].success) { - infobox_count = - add_infobox_to_collection(&infobox_data[i].result, &infobox_matrix, - &infobox_inner_counts, infobox_count); + add_infobox_to_collection(&infobox_data[i].result, &infobox_matrix); } } } - if (infobox_count > 0) { - context_set_array_of_arrays(&ctx, "infoboxes", infobox_matrix, - infobox_count, infobox_inner_counts); - free_string_matrix(infobox_matrix, infobox_inner_counts, infobox_count); + if (infobox_matrix.count > 0) { + context_set_array_of_arrays(&ctx, "infoboxes", infobox_matrix.rows, + infobox_matrix.count, + infobox_matrix.field_counts); } + string_matrix_free(&infobox_matrix); int warning_count = 0; for (int i = 0; i < enabled_engine_count; i++) { @@ -700,26 +696,25 @@ int results_handler(UrlParams *params) { } if (warning_count > 0) { - char ***warning_matrix = NULL; - int *warning_inner_counts = NULL; - int warning_index = 0; + StringMatrix warning_matrix; + string_matrix_init(&warning_matrix); for (int i = 0; i < enabled_engine_count; i++) { const char *warning_message = warning_message_for_job(&jobs[i], locale); if (!warning_message) continue; - warning_index = add_warning_to_collection( - jobs[i].engine->name, warning_message, &warning_matrix, - &warning_inner_counts, warning_index); + add_warning_to_collection(jobs[i].engine->name, warning_message, + &warning_matrix); } - if (warning_index > 0) { - context_set_array_of_arrays(&ctx, "engine_warnings", warning_matrix, - warning_index, warning_inner_counts); + if (warning_matrix.count > 0) { + context_set_array_of_arrays(&ctx, "engine_warnings", warning_matrix.rows, + warning_matrix.count, + warning_matrix.field_counts); } - free_string_matrix(warning_matrix, warning_inner_counts, warning_index); + string_matrix_free(&warning_matrix); } int total_results = 0; diff --git a/src/Scraping/Scraping.c b/src/Scraping/Scraping.c index 5c51ca5..9c4ff57 100644 --- a/src/Scraping/Scraping.c +++ b/src/Scraping/Scraping.c @@ -214,13 +214,17 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) { return -1; } - for (char *p = encoded_query + strlen(encoded_query) - 3; p >= encoded_query; - p--) { - if (p[0] == '%' && p[1] == '2' && p[2] == '0') { - *p = '+'; - memmove(p + 1, p + 3, strlen(p + 3) + 1); + char *read = encoded_query; + char *write = encoded_query; + while (*read) { + if (read[0] == '%' && read[1] == '2' && read[2] == '0') { + *write++ = '+'; + read += 3; + } else { + *write++ = *read++; } } + *write = '\0'; char *full_url = build_search_url(job->engine->base_url, job->engine->page_param, diff --git a/src/Utility/Utility.c b/src/Utility/Utility.c index 97a8d15..5183efe 100644 --- a/src/Utility/Utility.c +++ b/src/Utility/Utility.c @@ -14,7 +14,6 @@ static int themes_initialized = 0; void init_themes(const char *static_path) { if (themes_initialized) return; - themes_initialized = 1; char themes_dir[512]; snprintf(themes_dir, sizeof(themes_dir), "%s/themes", static_path); @@ -26,20 +25,32 @@ void init_themes(const char *static_path) { struct dirent *entry; int capacity = 4; themes_list = malloc(sizeof(char *) * capacity); + if (!themes_list) { + closedir(dir); + return; + } themes_count = 0; while ((entry = readdir(dir)) != NULL) { size_t len = strlen(entry->d_name); if (len > 4 && strcmp(entry->d_name + len - 4, ".css") == 0) { if (themes_count >= capacity) { - capacity *= 2; - themes_list = realloc(themes_list, sizeof(char *) * capacity); + int new_capacity = capacity * 2; + char **new_list = realloc(themes_list, sizeof(char *) * new_capacity); + if (!new_list) + break; + themes_list = new_list; + capacity = new_capacity; } - themes_list[themes_count] = strndup(entry->d_name, len - 4); + char *theme = strndup(entry->d_name, len - 4); + if (!theme) + break; + themes_list[themes_count] = theme; themes_count++; } } closedir(dir); + themes_initialized = 1; for (int i = 0; i < themes_count; i++) { for (int j = i + 1; j < themes_count; j++) { @@ -206,50 +217,6 @@ int user_engines_contains(const char *engine_id, char **ids, int count) { return 0; } -int append_string_row(char ****matrix, int **inner_counts, int row_count, - const char *const *values, int field_count) { - if (!matrix || !inner_counts || !values || field_count <= 0) - return row_count; - - char **row = calloc((size_t)field_count, sizeof(*row)); - if (!row) - return row_count; - - for (int i = 0; i < field_count; i++) { - row[i] = strdup(values[i] ? values[i] : ""); - if (!row[i]) { - for (int j = 0; j < i; j++) - free(row[j]); - free(row); - return row_count; - } - } - - char ***new_matrix = malloc(sizeof(*new_matrix) * (row_count + 1)); - int *new_counts = malloc(sizeof(*new_counts) * (row_count + 1)); - if (!new_matrix || !new_counts) { - free(new_matrix); - free(new_counts); - for (int i = 0; i < field_count; i++) - free(row[i]); - free(row); - return row_count; - } - - if (row_count > 0) { - memcpy(new_matrix, *matrix, sizeof(*new_matrix) * row_count); - memcpy(new_counts, *inner_counts, sizeof(*new_counts) * row_count); - } - - new_matrix[row_count] = row; - new_counts[row_count] = field_count; - free(*matrix); - free(*inner_counts); - *matrix = new_matrix; - *inner_counts = new_counts; - return row_count + 1; -} - void free_string_matrix(char ***matrix, int *inner_counts, int row_count) { if (matrix) { for (int i = 0; i < row_count; i++) { @@ -324,31 +291,6 @@ void string_matrix_free(StringMatrix *matrix) { *matrix = (StringMatrix){0}; } -int add_link_to_collection(const char *href, const char *label, - const char *class_name, char ****collection, - int **inner_counts, int current_count) { - const char *values[LINK_FIELD_COUNT] = {href, label, class_name}; - return append_string_row(collection, inner_counts, current_count, values, - LINK_FIELD_COUNT); -} - -int build_pagination(int page, char *(*href_builder)(int page, void *data), - void *data, char ****out_matrix, int **out_inner_counts) { - if (!out_matrix || !out_inner_counts) - return 0; - - StringMatrix matrix; - if (string_matrix_build_pagination(page, href_builder, data, &matrix) != 0) { - *out_matrix = NULL; - *out_inner_counts = NULL; - return 0; - } - - *out_matrix = matrix.rows; - *out_inner_counts = matrix.field_counts; - return matrix.count; -} - int string_matrix_build_pagination(int page, char *(*href_builder)(int page, void *data), void *data, StringMatrix *out_matrix) { diff --git a/src/Utility/Utility.h b/src/Utility/Utility.h index 663f2c4..a78fe65 100644 --- a/src/Utility/Utility.h +++ b/src/Utility/Utility.h @@ -31,8 +31,6 @@ int is_engine_id_enabled(const char *engine_id); int get_user_engines(char ***out_ids, int *out_count); int user_engines_contains(const char *engine_id, char **ids, int count); -int append_string_row(char ****matrix, int **inner_counts, int row_count, - const char *const *values, int field_count); void free_string_matrix(char ***matrix, int *inner_counts, int row_count); void string_matrix_init(StringMatrix *matrix); @@ -40,12 +38,6 @@ int string_matrix_append(StringMatrix *matrix, const char *const *values, int field_count); void string_matrix_free(StringMatrix *matrix); -int add_link_to_collection(const char *href, const char *label, - const char *class_name, char ****collection, - int **inner_counts, int current_count); - -int build_pagination(int page, char *(*href_builder)(int page, void *data), - void *data, char ****out_matrix, int **out_inner_counts); int string_matrix_build_pagination(int page, char *(*href_builder)(int page, void *data), void *data, StringMatrix *out_matrix); -- cgit v1.3 From 68d9d46e5ee5eeefca8630ee4c26f7794a8842a4 Mon Sep 17 00:00:00 2001 From: frosty Date: Sat, 15 Aug 2026 02:57:29 -0400 Subject: fix: improve memory handling in search cleanup paths --- src/Routes/Search.c | 3 +- src/Scraping/Scraping.c | 85 ++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 67 insertions(+), 21 deletions(-) (limited to 'src/Scraping') diff --git a/src/Routes/Search.c b/src/Routes/Search.c index e7703be..7fcdac8 100644 --- a/src/Routes/Search.c +++ b/src/Routes/Search.c @@ -287,8 +287,7 @@ static void free_search_results(SearchResult **all_results, ScrapeJob *jobs, static void free_infobox_results(InfoBoxThreadData *data) { for (int i = 0; i < HANDLER_COUNT; i++) { - if (data[i].success) - free_infobox(&data[i].result); + free_infobox(&data[i].result); } } diff --git a/src/Scraping/Scraping.c b/src/Scraping/Scraping.c index 9c4ff57..6d94450 100644 --- a/src/Scraping/Scraping.c +++ b/src/Scraping/Scraping.c @@ -1,6 +1,7 @@ #include "Scraping.h" #include "../Cache/Cache.h" #include "../Proxy/Proxy.h" +#include "../Utility/XmlHelper.h" #include "Config.h" #include #include @@ -170,14 +171,26 @@ void parse_and_cache_response(ScrapeJob *job) { } } -void cleanup_job_handle(ScrapeJob *job, CURL *handle) { +static void cleanup_job_results(ScrapeJob *job) { + if (!job || !job->out_results) + return; + + xml_result_free(*job->out_results, job->results_count); + *job->out_results = NULL; + job->results_count = 0; +} + +static void cleanup_job_handle(ScrapeJob *job, CURL *handle) { struct curl_slist *headers = NULL; - curl_easy_getinfo(handle, CURLINFO_PRIVATE, &headers); + if (handle) + curl_easy_getinfo(handle, CURLINFO_PRIVATE, &headers); if (headers) curl_slist_free_all(headers); free(job->response.memory); job->response.memory = NULL; + job->response.size = 0; + job->response.capacity = 0; } void process_response(ScrapeJob *job, CURL *handle, CURLMsg *msg) { @@ -194,19 +207,25 @@ void process_response(ScrapeJob *job, CURL *handle, CURLMsg *msg) { } int setup_job(ScrapeJob *job, CURLM *multi_handle) { - if (job->handle) + if (job->handle) { + cleanup_job_handle(job, job->handle); curl_easy_cleanup(job->handle); - if (job->response.memory) + job->handle = NULL; + } else if (job->response.memory) { free(job->response.memory); + job->response.memory = NULL; + job->response.size = 0; + job->response.capacity = 0; + } - job->results_count = 0; + cleanup_job_results(job); job->http_status = 0; job->status = SCRAPE_STATUS_PENDING; - if (check_cache_for_job(job)) { - job->results_count = job->results_count > 0 ? job->results_count : 0; + if (check_cache_for_job(job)) return 0; - } + + cleanup_job_results(job); char *encoded_query = curl_easy_escape(NULL, job->query, 0); if (!encoded_query) { @@ -247,6 +266,14 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) { job->response.memory = (char *)malloc(INITIAL_BUFFER_SIZE); job->response.size = 0; job->response.capacity = INITIAL_BUFFER_SIZE; + if (!job->response.memory) { + curl_easy_cleanup(job->handle); + job->handle = NULL; + job->response.capacity = 0; + free(full_url); + job->status = SCRAPE_STATUS_FETCH_ERROR; + return -1; + } struct curl_slist *headers = build_request_headers(job->engine->host_header, job->engine->referer); @@ -255,10 +282,33 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) { curl_easy_setopt(job->handle, CURLOPT_PRIVATE, headers); free(full_url); - curl_multi_add_handle(multi_handle, job->handle); + CURLMcode add_result = curl_multi_add_handle(multi_handle, job->handle); + if (add_result != CURLM_OK) { + cleanup_job_handle(job, job->handle); + curl_easy_cleanup(job->handle); + job->handle = NULL; + job->status = SCRAPE_STATUS_FETCH_ERROR; + return -1; + } return 0; } +static void cleanup_unfinished_jobs(CURLM *multi_handle, ScrapeJob *jobs, + int num_jobs) { + for (int i = 0; i < num_jobs; i++) { + if (!jobs[i].handle) + continue; + + curl_multi_remove_handle(multi_handle, jobs[i].handle); + cleanup_job_handle(&jobs[i], jobs[i].handle); + curl_easy_cleanup(jobs[i].handle); + jobs[i].handle = NULL; + + if (jobs[i].status == SCRAPE_STATUS_PENDING) + jobs[i].status = SCRAPE_STATUS_FETCH_ERROR; + } +} + int handle_responses(CURLM *multi_handle, ScrapeJob *jobs, int num_jobs) { CURLMsg *msg; int msgs_left; @@ -304,27 +354,24 @@ retry:; return -1; for (int i = 0; i < num_jobs; i++) { - if (setup_job(&jobs[i], multi_handle) != 0 && jobs[i].handle) { - curl_multi_remove_handle(multi_handle, jobs[i].handle); - curl_easy_cleanup(jobs[i].handle); - jobs[i].handle = NULL; - } + setup_job(&jobs[i], multi_handle); } http_delay(); int still_running = 0; - curl_multi_perform(multi_handle, &still_running); + CURLMcode mc = curl_multi_perform(multi_handle, &still_running); - do { + while (mc == CURLM_OK && still_running) { int numfds = 0; - CURLMcode mc = curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds); + mc = curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds); if (mc != CURLM_OK) break; - curl_multi_perform(multi_handle, &still_running); - } while (still_running); + mc = curl_multi_perform(multi_handle, &still_running); + } handle_responses(multi_handle, jobs, num_jobs); + cleanup_unfinished_jobs(multi_handle, jobs, num_jobs); curl_multi_cleanup(multi_handle); if (retries < max_proxy_retries && should_retry(jobs, num_jobs)) { -- cgit v1.3