diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/Config.c | 4 | ||||
| -rw-r--r-- | src/Config.h | 2 | ||||
| -rw-r--r-- | src/Main.c | 18 | ||||
| -rw-r--r-- | src/Scraping/Scraping.c | 36 | ||||
| -rw-r--r-- | src/Scraping/Scraping.h | 2 | ||||
| -rw-r--r-- | src/Scraping/ScrapingHttp.c | 13 | ||||
| -rw-r--r-- | src/Scraping/ScrapingParsers.c | 116 |
7 files changed, 166 insertions, 25 deletions
diff --git a/src/Config.c b/src/Config.c index 9883d45..bde76fe 100644 --- a/src/Config.c +++ b/src/Config.c @@ -100,6 +100,10 @@ int load_config(const char *filename, Config *config) { if (strcmp(key, "engines") == 0) { strncpy(config->engines, value, sizeof(config->engines) - 1); config->engines[sizeof(config->engines) - 1] = '\0'; + } else if (strcmp(key, "yacy_instance") == 0) { + strncpy(config->yacy_instance, value, + sizeof(config->yacy_instance) - 1); + config->yacy_instance[sizeof(config->yacy_instance) - 1] = '\0'; } } else if (strcmp(section, "rate_limit") == 0) { if (strcmp(key, "search_requests") == 0) { diff --git a/src/Config.h b/src/Config.h index 25bd978..f0bdf4b 100644 --- a/src/Config.h +++ b/src/Config.h @@ -8,6 +8,7 @@ #define DEFAULT_CACHE_TTL_INFOBOX 86400 #define DEFAULT_CACHE_TTL_IMAGE 604800 #define DEFAULT_MAX_PROXY_RETRIES 3 +#define DEFAULT_YACY_INSTANCE "http://127.0.0.1:8090" #define BUFFER_SIZE_SMALL 256 #define BUFFER_SIZE_MEDIUM 512 @@ -46,6 +47,7 @@ typedef struct { int cache_ttl_infobox; int cache_ttl_image; char engines[512]; + char yacy_instance[512]; int rate_limit_search_requests; int rate_limit_search_interval; int rate_limit_images_requests; @@ -78,6 +78,7 @@ int main() { .cache_ttl_infobox = DEFAULT_CACHE_TTL_INFOBOX, .cache_ttl_image = DEFAULT_CACHE_TTL_IMAGE, .engines = "", + .yacy_instance = "", .rate_limit_search_requests = 0, .rate_limit_search_interval = 0, .rate_limit_images_requests = 0, @@ -101,6 +102,23 @@ int main() { } apply_engines_config(cfg.engines); + configure_yacy_engine(cfg.yacy_instance); + + if (cfg.yacy_instance[0] != '\0') { + int yacy_enabled = 0; + for (int i = 0; i < ENGINE_COUNT; i++) { + if (strcmp(ENGINE_REGISTRY[i].id, "yacy") == 0) { + yacy_enabled = ENGINE_REGISTRY[i].enabled; + break; + } + } + if (!yacy_enabled) { + fprintf( + stderr, + "[INFO] YaCy instance configured but the yacy engine is not " + "enabled (add it to the engines list, e.g. engines=\"*,yacy\")\n"); + } + } if (cache_init(cfg.cache_dir) != 0) { fprintf(stderr, diff --git a/src/Scraping/Scraping.c b/src/Scraping/Scraping.c index b81c216..5c51ca5 100644 --- a/src/Scraping/Scraping.c +++ b/src/Scraping/Scraping.c @@ -4,6 +4,7 @@ #include "Config.h" #include <curl/curl.h> #include <libxml/HTMLparser.h> +#include <libxml/parser.h> #include <stdio.h> #include <stdlib.h> #include <string.h> @@ -24,7 +25,8 @@ static int response_is_startpage_captcha(const ScrapeJob *job, return response_contains(response, "<title>Startpage Captcha</title>") || response_contains(response, "Startpage Captcha") || - response_contains(response, "/static-pages-assets/page-data/captcha/") || + response_contains(response, + "/static-pages-assets/page-data/captcha/") || response_contains(response, ">Startpage Blocked</title>"); } @@ -39,7 +41,8 @@ static int response_looks_like_results_page(const ScrapeJob *job, } if (strcmp(job->engine->name, "Startpage") == 0) { - return response_contains(response, "<title>Startpage Search Results</title>") || + return response_contains(response, + "<title>Startpage Search Results</title>") || response_contains(response, "class=\"w-gl") || response_contains(response, "data-testid=\"gl-title-link\""); } @@ -55,6 +58,11 @@ static int response_looks_like_results_page(const ScrapeJob *job, response_contains(response, "Mojeek Search"); } + if (strcmp(job->engine->name, "YaCy") == 0) { + return response_contains(response, "<item") || + response_contains(response, "<rss"); + } + return 0; } @@ -72,18 +80,24 @@ static void classify_job_response(ScrapeJob *job, const char *response, return; } - xmlDocPtr doc = htmlReadMemory(response, response_size, NULL, NULL, - HTML_PARSE_RECOVER | HTML_PARSE_NOERROR | - HTML_PARSE_NOWARNING); + xmlDocPtr doc; + if (job->engine->is_xml) { + doc = xmlReadMemory(response, response_size, NULL, NULL, + XML_PARSE_RECOVER | XML_PARSE_NOERROR | + XML_PARSE_NOWARNING); + } else { + doc = htmlReadMemory(response, response_size, NULL, NULL, + HTML_PARSE_RECOVER | HTML_PARSE_NOERROR | + HTML_PARSE_NOWARNING); + } if (!doc) { job->status = SCRAPE_STATUS_FETCH_ERROR; return; } - job->results_count = - job->engine->parser(job->engine->name, doc, job->out_results, - job->max_results); + job->results_count = job->engine->parser(job->engine->name, doc, + job->out_results, job->max_results); xmlFreeDoc(doc); if (job->results_count > 0) { @@ -200,7 +214,8 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) { return -1; } - for (char *p = encoded_query + strlen(encoded_query) - 3; p >= encoded_query; p--) { + for (char *p = encoded_query + strlen(encoded_query) - 3; p >= encoded_query; + p--) { if (p[0] == '%' && p[1] == '2' && p[2] == '0') { *p = '+'; memmove(p + 1, p + 3, strlen(p + 3) + 1); @@ -279,8 +294,7 @@ int should_retry(ScrapeJob *jobs, int num_jobs) { int scrape_engines_parallel(ScrapeJob *jobs, int num_jobs) { int retries = 0; -retry: - ; +retry:; CURLM *multi_handle = curl_multi_init(); if (!multi_handle) return -1; diff --git a/src/Scraping/Scraping.h b/src/Scraping/Scraping.h index be65e5a..14e6e55 100644 --- a/src/Scraping/Scraping.h +++ b/src/Scraping/Scraping.h @@ -26,6 +26,7 @@ typedef struct { int page_base; ParserFunc parser; int enabled; + int is_xml; } SearchEngine; typedef struct { @@ -59,6 +60,7 @@ typedef struct { extern SearchEngine ENGINE_REGISTRY[]; extern const int ENGINE_COUNT; void apply_engines_config(const char *engines_str); +void configure_yacy_engine(const char *instance); size_t write_memory_callback(void *contents, size_t size, size_t nmemb, void *userp); diff --git a/src/Scraping/ScrapingHttp.c b/src/Scraping/ScrapingHttp.c index 1a6a292..89f4853 100644 --- a/src/Scraping/ScrapingHttp.c +++ b/src/Scraping/ScrapingHttp.c @@ -89,11 +89,14 @@ struct curl_slist *build_request_headers(const char *host_header, struct curl_slist *headers = NULL; char host_buf[BUFFER_SIZE_MEDIUM], ref_buf[BUFFER_SIZE_MEDIUM]; - snprintf(host_buf, sizeof(host_buf), "Host: %s", host_header); - snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer); - - headers = curl_slist_append(headers, host_buf); - headers = curl_slist_append(headers, ref_buf); + if (host_header && host_header[0] != '\0') { + snprintf(host_buf, sizeof(host_buf), "Host: %s", host_header); + headers = curl_slist_append(headers, host_buf); + } + if (referer && referer[0] != '\0') { + snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer); + headers = curl_slist_append(headers, ref_buf); + } headers = curl_slist_append( headers, "Accept: " diff --git a/src/Scraping/ScrapingParsers.c b/src/Scraping/ScrapingParsers.c index 96aaded..2e5689b 100644 --- a/src/Scraping/ScrapingParsers.c +++ b/src/Scraping/ScrapingParsers.c @@ -249,8 +249,8 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc, if (!ctx) return 0; - xmlXPathObjectPtr obj = - xml_xpath_eval(ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]"); + xmlXPathObjectPtr obj = xml_xpath_eval( + ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]"); if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) { free_xpath_objects(ctx, obj); @@ -268,18 +268,17 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc, xmlNodePtr result_node = obj->nodesetval->nodeTab[i]; ctx->node = result_node; - xmlXPathObjectPtr link_obj = - xml_xpath_eval(ctx, ".//a[@class='title']"); + xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, ".//a[@class='title']"); char *url = (link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0) ? (char *)xmlGetProp(link_obj->nodesetval->nodeTab[0], (xmlChar *)"href") : NULL; - char *title = (link_obj && link_obj->nodesetval && - link_obj->nodesetval->nodeNr > 0) - ? xml_node_content(link_obj->nodesetval->nodeTab[0]) - : NULL; + char *title = + (link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0) + ? xml_node_content(link_obj->nodesetval->nodeTab[0]) + : NULL; xmlXPathObjectPtr snippet_obj = xml_xpath_eval(ctx, ".//p[@class='s']"); char *snippet_text = @@ -310,6 +309,90 @@ static int parse_yahoo(const char *engine_name, xmlDocPtr doc, static int parse_mojeek(const char *engine_name, xmlDocPtr doc, SearchResult **out_results, int max_results); +static int parse_yacy(const char *engine_name, xmlDocPtr doc, + SearchResult **out_results, int max_results) { + (void)engine_name; + int found_count = 0; + + xmlXPathContextPtr ctx = create_xpath_context(doc); + if (!ctx) + return 0; + + xmlXPathObjectPtr obj = xml_xpath_eval(ctx, "//item"); + + if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) { + free_xpath_objects(ctx, obj); + return 0; + } + + int num_items = obj->nodesetval->nodeNr; + *out_results = alloc_results_array(num_items, max_results); + if (!*out_results) { + free_xpath_objects(ctx, obj); + return 0; + } + + for (int i = 0; i < num_items && found_count < max_results; i++) { + xmlNodePtr item_node = obj->nodesetval->nodeTab[i]; + ctx->node = item_node; + + xmlXPathObjectPtr title_obj = xml_xpath_eval(ctx, "./title"); + char *title = (title_obj && title_obj->nodesetval && + title_obj->nodesetval->nodeNr > 0) + ? xml_node_content(title_obj->nodesetval->nodeTab[0]) + : NULL; + + xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, "./link"); + char *url = + (link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0) + ? xml_node_content(link_obj->nodesetval->nodeTab[0]) + : NULL; + + xmlXPathObjectPtr desc_obj = xml_xpath_eval(ctx, "./description"); + char *snippet_text = + (desc_obj && desc_obj->nodesetval && desc_obj->nodesetval->nodeNr > 0) + ? xml_node_content(desc_obj->nodesetval->nodeTab[0]) + : NULL; + + if (url && title) { + assign_result(&(*out_results)[found_count], url, title, snippet_text, 0); + found_count++; + } + + free_xml_node_list(title, url, snippet_text); + if (title_obj) + xmlXPathFreeObject(title_obj); + if (link_obj) + xmlXPathFreeObject(link_obj); + if (desc_obj) + xmlXPathFreeObject(desc_obj); + } + + ctx->node = NULL; + free_xpath_objects(ctx, obj); + return found_count; +} + +static char yacy_base_url[BUFFER_SIZE_LARGE]; + +void configure_yacy_engine(const char *instance) { + if (!instance || instance[0] == '\0') + instance = DEFAULT_YACY_INSTANCE; + + snprintf(yacy_base_url, sizeof(yacy_base_url), + "%s/" + "yacysearch.rss?resource=global&urlmaskfilter=.*&prefermaskfilter=&" + "nav=all&maximumRecords=%d&query=", + instance, MAX_RESULTS_PER_ENGINE); + + for (int i = 0; i < ENGINE_COUNT; i++) { + if (strcmp(ENGINE_REGISTRY[i].id, "yacy") == 0) { + ENGINE_REGISTRY[i].base_url = yacy_base_url; + break; + } + } +} + SearchEngine ENGINE_REGISTRY[] = { {.id = "ddg", .name = "DuckDuckGo Lite", @@ -350,7 +433,18 @@ SearchEngine ENGINE_REGISTRY[] = { .page_multiplier = 10, .page_base = 1, .parser = parse_mojeek, - .enabled = 1}}; + .enabled = 1}, + {.id = "yacy", + .name = "YaCy", + .base_url = "", + .host_header = NULL, + .referer = NULL, + .page_param = "startRecord", + .page_multiplier = 10, + .page_base = 0, + .parser = parse_yacy, + .enabled = 0, + .is_xml = 1}}; const int ENGINE_COUNT = sizeof(ENGINE_REGISTRY) / sizeof(SearchEngine); @@ -373,6 +467,8 @@ static int engine_id_compare(const char *engine_id, const char *config_id) { void apply_engines_config(const char *engines_str) { if (!engines_str || engines_str[0] == '\0') { for (int i = 0; i < ENGINE_COUNT; i++) { + if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy")) + continue; ENGINE_REGISTRY[i].enabled = 1; } return; @@ -395,6 +491,8 @@ void apply_engines_config(const char *engines_str) { if (strcmp(token, "*") == 0) { for (int i = 0; i < ENGINE_COUNT; i++) { + if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy")) + continue; ENGINE_REGISTRY[i].enabled = 1; } } else if (token[0] == '-' && token[1] != '\0') { |