aboutsummaryrefslogtreecommitdiff
path: root/src/Scraping
diff options
context:
space:
mode:
Diffstat (limited to 'src/Scraping')
-rw-r--r--src/Scraping/Scraping.c131
-rw-r--r--src/Scraping/Scraping.h2
-rw-r--r--src/Scraping/ScrapingHttp.c13
-rw-r--r--src/Scraping/ScrapingParsers.c116
4 files changed, 215 insertions, 47 deletions
diff --git a/src/Scraping/Scraping.c b/src/Scraping/Scraping.c
index b81c216..6d94450 100644
--- a/src/Scraping/Scraping.c
+++ b/src/Scraping/Scraping.c
@@ -1,9 +1,11 @@
#include "Scraping.h"
#include "../Cache/Cache.h"
#include "../Proxy/Proxy.h"
+#include "../Utility/XmlHelper.h"
#include "Config.h"
#include <curl/curl.h>
#include <libxml/HTMLparser.h>
+#include <libxml/parser.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -24,7 +26,8 @@ static int response_is_startpage_captcha(const ScrapeJob *job,
return response_contains(response, "<title>Startpage Captcha</title>") ||
response_contains(response, "Startpage Captcha") ||
- response_contains(response, "/static-pages-assets/page-data/captcha/") ||
+ response_contains(response,
+ "/static-pages-assets/page-data/captcha/") ||
response_contains(response, ">Startpage Blocked</title>");
}
@@ -39,7 +42,8 @@ static int response_looks_like_results_page(const ScrapeJob *job,
}
if (strcmp(job->engine->name, "Startpage") == 0) {
- return response_contains(response, "<title>Startpage Search Results</title>") ||
+ return response_contains(response,
+ "<title>Startpage Search Results</title>") ||
response_contains(response, "class=\"w-gl") ||
response_contains(response, "data-testid=\"gl-title-link\"");
}
@@ -55,6 +59,11 @@ static int response_looks_like_results_page(const ScrapeJob *job,
response_contains(response, "Mojeek Search");
}
+ if (strcmp(job->engine->name, "YaCy") == 0) {
+ return response_contains(response, "<item") ||
+ response_contains(response, "<rss");
+ }
+
return 0;
}
@@ -72,18 +81,24 @@ static void classify_job_response(ScrapeJob *job, const char *response,
return;
}
- xmlDocPtr doc = htmlReadMemory(response, response_size, NULL, NULL,
- HTML_PARSE_RECOVER | HTML_PARSE_NOERROR |
- HTML_PARSE_NOWARNING);
+ xmlDocPtr doc;
+ if (job->engine->is_xml) {
+ doc = xmlReadMemory(response, response_size, NULL, NULL,
+ XML_PARSE_RECOVER | XML_PARSE_NOERROR |
+ XML_PARSE_NOWARNING);
+ } else {
+ doc = htmlReadMemory(response, response_size, NULL, NULL,
+ HTML_PARSE_RECOVER | HTML_PARSE_NOERROR |
+ HTML_PARSE_NOWARNING);
+ }
if (!doc) {
job->status = SCRAPE_STATUS_FETCH_ERROR;
return;
}
- job->results_count =
- job->engine->parser(job->engine->name, doc, job->out_results,
- job->max_results);
+ job->results_count = job->engine->parser(job->engine->name, doc,
+ job->out_results, job->max_results);
xmlFreeDoc(doc);
if (job->results_count > 0) {
@@ -156,14 +171,26 @@ void parse_and_cache_response(ScrapeJob *job) {
}
}
-void cleanup_job_handle(ScrapeJob *job, CURL *handle) {
+static void cleanup_job_results(ScrapeJob *job) {
+ if (!job || !job->out_results)
+ return;
+
+ xml_result_free(*job->out_results, job->results_count);
+ *job->out_results = NULL;
+ job->results_count = 0;
+}
+
+static void cleanup_job_handle(ScrapeJob *job, CURL *handle) {
struct curl_slist *headers = NULL;
- curl_easy_getinfo(handle, CURLINFO_PRIVATE, &headers);
+ if (handle)
+ curl_easy_getinfo(handle, CURLINFO_PRIVATE, &headers);
if (headers)
curl_slist_free_all(headers);
free(job->response.memory);
job->response.memory = NULL;
+ job->response.size = 0;
+ job->response.capacity = 0;
}
void process_response(ScrapeJob *job, CURL *handle, CURLMsg *msg) {
@@ -180,19 +207,25 @@ void process_response(ScrapeJob *job, CURL *handle, CURLMsg *msg) {
}
int setup_job(ScrapeJob *job, CURLM *multi_handle) {
- if (job->handle)
+ if (job->handle) {
+ cleanup_job_handle(job, job->handle);
curl_easy_cleanup(job->handle);
- if (job->response.memory)
+ job->handle = NULL;
+ } else if (job->response.memory) {
free(job->response.memory);
+ job->response.memory = NULL;
+ job->response.size = 0;
+ job->response.capacity = 0;
+ }
- job->results_count = 0;
+ cleanup_job_results(job);
job->http_status = 0;
job->status = SCRAPE_STATUS_PENDING;
- if (check_cache_for_job(job)) {
- job->results_count = job->results_count > 0 ? job->results_count : 0;
+ if (check_cache_for_job(job))
return 0;
- }
+
+ cleanup_job_results(job);
char *encoded_query = curl_easy_escape(NULL, job->query, 0);
if (!encoded_query) {
@@ -200,12 +233,17 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) {
return -1;
}
- for (char *p = encoded_query + strlen(encoded_query) - 3; p >= encoded_query; p--) {
- if (p[0] == '%' && p[1] == '2' && p[2] == '0') {
- *p = '+';
- memmove(p + 1, p + 3, strlen(p + 3) + 1);
+ char *read = encoded_query;
+ char *write = encoded_query;
+ while (*read) {
+ if (read[0] == '%' && read[1] == '2' && read[2] == '0') {
+ *write++ = '+';
+ read += 3;
+ } else {
+ *write++ = *read++;
}
}
+ *write = '\0';
char *full_url =
build_search_url(job->engine->base_url, job->engine->page_param,
@@ -228,6 +266,14 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) {
job->response.memory = (char *)malloc(INITIAL_BUFFER_SIZE);
job->response.size = 0;
job->response.capacity = INITIAL_BUFFER_SIZE;
+ if (!job->response.memory) {
+ curl_easy_cleanup(job->handle);
+ job->handle = NULL;
+ job->response.capacity = 0;
+ free(full_url);
+ job->status = SCRAPE_STATUS_FETCH_ERROR;
+ return -1;
+ }
struct curl_slist *headers =
build_request_headers(job->engine->host_header, job->engine->referer);
@@ -236,10 +282,33 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) {
curl_easy_setopt(job->handle, CURLOPT_PRIVATE, headers);
free(full_url);
- curl_multi_add_handle(multi_handle, job->handle);
+ CURLMcode add_result = curl_multi_add_handle(multi_handle, job->handle);
+ if (add_result != CURLM_OK) {
+ cleanup_job_handle(job, job->handle);
+ curl_easy_cleanup(job->handle);
+ job->handle = NULL;
+ job->status = SCRAPE_STATUS_FETCH_ERROR;
+ return -1;
+ }
return 0;
}
+static void cleanup_unfinished_jobs(CURLM *multi_handle, ScrapeJob *jobs,
+ int num_jobs) {
+ for (int i = 0; i < num_jobs; i++) {
+ if (!jobs[i].handle)
+ continue;
+
+ curl_multi_remove_handle(multi_handle, jobs[i].handle);
+ cleanup_job_handle(&jobs[i], jobs[i].handle);
+ curl_easy_cleanup(jobs[i].handle);
+ jobs[i].handle = NULL;
+
+ if (jobs[i].status == SCRAPE_STATUS_PENDING)
+ jobs[i].status = SCRAPE_STATUS_FETCH_ERROR;
+ }
+}
+
int handle_responses(CURLM *multi_handle, ScrapeJob *jobs, int num_jobs) {
CURLMsg *msg;
int msgs_left;
@@ -279,34 +348,30 @@ int should_retry(ScrapeJob *jobs, int num_jobs) {
int scrape_engines_parallel(ScrapeJob *jobs, int num_jobs) {
int retries = 0;
-retry:
- ;
+retry:;
CURLM *multi_handle = curl_multi_init();
if (!multi_handle)
return -1;
for (int i = 0; i < num_jobs; i++) {
- if (setup_job(&jobs[i], multi_handle) != 0 && jobs[i].handle) {
- curl_multi_remove_handle(multi_handle, jobs[i].handle);
- curl_easy_cleanup(jobs[i].handle);
- jobs[i].handle = NULL;
- }
+ setup_job(&jobs[i], multi_handle);
}
http_delay();
int still_running = 0;
- curl_multi_perform(multi_handle, &still_running);
+ CURLMcode mc = curl_multi_perform(multi_handle, &still_running);
- do {
+ while (mc == CURLM_OK && still_running) {
int numfds = 0;
- CURLMcode mc = curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds);
+ mc = curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds);
if (mc != CURLM_OK)
break;
- curl_multi_perform(multi_handle, &still_running);
- } while (still_running);
+ mc = curl_multi_perform(multi_handle, &still_running);
+ }
handle_responses(multi_handle, jobs, num_jobs);
+ cleanup_unfinished_jobs(multi_handle, jobs, num_jobs);
curl_multi_cleanup(multi_handle);
if (retries < max_proxy_retries && should_retry(jobs, num_jobs)) {
diff --git a/src/Scraping/Scraping.h b/src/Scraping/Scraping.h
index be65e5a..14e6e55 100644
--- a/src/Scraping/Scraping.h
+++ b/src/Scraping/Scraping.h
@@ -26,6 +26,7 @@ typedef struct {
int page_base;
ParserFunc parser;
int enabled;
+ int is_xml;
} SearchEngine;
typedef struct {
@@ -59,6 +60,7 @@ typedef struct {
extern SearchEngine ENGINE_REGISTRY[];
extern const int ENGINE_COUNT;
void apply_engines_config(const char *engines_str);
+void configure_yacy_engine(const char *instance);
size_t write_memory_callback(void *contents, size_t size, size_t nmemb,
void *userp);
diff --git a/src/Scraping/ScrapingHttp.c b/src/Scraping/ScrapingHttp.c
index 1a6a292..89f4853 100644
--- a/src/Scraping/ScrapingHttp.c
+++ b/src/Scraping/ScrapingHttp.c
@@ -89,11 +89,14 @@ struct curl_slist *build_request_headers(const char *host_header,
struct curl_slist *headers = NULL;
char host_buf[BUFFER_SIZE_MEDIUM], ref_buf[BUFFER_SIZE_MEDIUM];
- snprintf(host_buf, sizeof(host_buf), "Host: %s", host_header);
- snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer);
-
- headers = curl_slist_append(headers, host_buf);
- headers = curl_slist_append(headers, ref_buf);
+ if (host_header && host_header[0] != '\0') {
+ snprintf(host_buf, sizeof(host_buf), "Host: %s", host_header);
+ headers = curl_slist_append(headers, host_buf);
+ }
+ if (referer && referer[0] != '\0') {
+ snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer);
+ headers = curl_slist_append(headers, ref_buf);
+ }
headers = curl_slist_append(
headers,
"Accept: "
diff --git a/src/Scraping/ScrapingParsers.c b/src/Scraping/ScrapingParsers.c
index 96aaded..2e5689b 100644
--- a/src/Scraping/ScrapingParsers.c
+++ b/src/Scraping/ScrapingParsers.c
@@ -249,8 +249,8 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
if (!ctx)
return 0;
- xmlXPathObjectPtr obj =
- xml_xpath_eval(ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]");
+ xmlXPathObjectPtr obj = xml_xpath_eval(
+ ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]");
if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) {
free_xpath_objects(ctx, obj);
@@ -268,18 +268,17 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
xmlNodePtr result_node = obj->nodesetval->nodeTab[i];
ctx->node = result_node;
- xmlXPathObjectPtr link_obj =
- xml_xpath_eval(ctx, ".//a[@class='title']");
+ xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, ".//a[@class='title']");
char *url =
(link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0)
? (char *)xmlGetProp(link_obj->nodesetval->nodeTab[0],
(xmlChar *)"href")
: NULL;
- char *title = (link_obj && link_obj->nodesetval &&
- link_obj->nodesetval->nodeNr > 0)
- ? xml_node_content(link_obj->nodesetval->nodeTab[0])
- : NULL;
+ char *title =
+ (link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0)
+ ? xml_node_content(link_obj->nodesetval->nodeTab[0])
+ : NULL;
xmlXPathObjectPtr snippet_obj = xml_xpath_eval(ctx, ".//p[@class='s']");
char *snippet_text =
@@ -310,6 +309,90 @@ static int parse_yahoo(const char *engine_name, xmlDocPtr doc,
static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
SearchResult **out_results, int max_results);
+static int parse_yacy(const char *engine_name, xmlDocPtr doc,
+ SearchResult **out_results, int max_results) {
+ (void)engine_name;
+ int found_count = 0;
+
+ xmlXPathContextPtr ctx = create_xpath_context(doc);
+ if (!ctx)
+ return 0;
+
+ xmlXPathObjectPtr obj = xml_xpath_eval(ctx, "//item");
+
+ if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) {
+ free_xpath_objects(ctx, obj);
+ return 0;
+ }
+
+ int num_items = obj->nodesetval->nodeNr;
+ *out_results = alloc_results_array(num_items, max_results);
+ if (!*out_results) {
+ free_xpath_objects(ctx, obj);
+ return 0;
+ }
+
+ for (int i = 0; i < num_items && found_count < max_results; i++) {
+ xmlNodePtr item_node = obj->nodesetval->nodeTab[i];
+ ctx->node = item_node;
+
+ xmlXPathObjectPtr title_obj = xml_xpath_eval(ctx, "./title");
+ char *title = (title_obj && title_obj->nodesetval &&
+ title_obj->nodesetval->nodeNr > 0)
+ ? xml_node_content(title_obj->nodesetval->nodeTab[0])
+ : NULL;
+
+ xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, "./link");
+ char *url =
+ (link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0)
+ ? xml_node_content(link_obj->nodesetval->nodeTab[0])
+ : NULL;
+
+ xmlXPathObjectPtr desc_obj = xml_xpath_eval(ctx, "./description");
+ char *snippet_text =
+ (desc_obj && desc_obj->nodesetval && desc_obj->nodesetval->nodeNr > 0)
+ ? xml_node_content(desc_obj->nodesetval->nodeTab[0])
+ : NULL;
+
+ if (url && title) {
+ assign_result(&(*out_results)[found_count], url, title, snippet_text, 0);
+ found_count++;
+ }
+
+ free_xml_node_list(title, url, snippet_text);
+ if (title_obj)
+ xmlXPathFreeObject(title_obj);
+ if (link_obj)
+ xmlXPathFreeObject(link_obj);
+ if (desc_obj)
+ xmlXPathFreeObject(desc_obj);
+ }
+
+ ctx->node = NULL;
+ free_xpath_objects(ctx, obj);
+ return found_count;
+}
+
+static char yacy_base_url[BUFFER_SIZE_LARGE];
+
+void configure_yacy_engine(const char *instance) {
+ if (!instance || instance[0] == '\0')
+ instance = DEFAULT_YACY_INSTANCE;
+
+ snprintf(yacy_base_url, sizeof(yacy_base_url),
+ "%s/"
+ "yacysearch.rss?resource=global&urlmaskfilter=.*&prefermaskfilter=&"
+ "nav=all&maximumRecords=%d&query=",
+ instance, MAX_RESULTS_PER_ENGINE);
+
+ for (int i = 0; i < ENGINE_COUNT; i++) {
+ if (strcmp(ENGINE_REGISTRY[i].id, "yacy") == 0) {
+ ENGINE_REGISTRY[i].base_url = yacy_base_url;
+ break;
+ }
+ }
+}
+
SearchEngine ENGINE_REGISTRY[] = {
{.id = "ddg",
.name = "DuckDuckGo Lite",
@@ -350,7 +433,18 @@ SearchEngine ENGINE_REGISTRY[] = {
.page_multiplier = 10,
.page_base = 1,
.parser = parse_mojeek,
- .enabled = 1}};
+ .enabled = 1},
+ {.id = "yacy",
+ .name = "YaCy",
+ .base_url = "",
+ .host_header = NULL,
+ .referer = NULL,
+ .page_param = "startRecord",
+ .page_multiplier = 10,
+ .page_base = 0,
+ .parser = parse_yacy,
+ .enabled = 0,
+ .is_xml = 1}};
const int ENGINE_COUNT = sizeof(ENGINE_REGISTRY) / sizeof(SearchEngine);
@@ -373,6 +467,8 @@ static int engine_id_compare(const char *engine_id, const char *config_id) {
void apply_engines_config(const char *engines_str) {
if (!engines_str || engines_str[0] == '\0') {
for (int i = 0; i < ENGINE_COUNT; i++) {
+ if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy"))
+ continue;
ENGINE_REGISTRY[i].enabled = 1;
}
return;
@@ -395,6 +491,8 @@ void apply_engines_config(const char *engines_str) {
if (strcmp(token, "*") == 0) {
for (int i = 0; i < ENGINE_COUNT; i++) {
+ if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy"))
+ continue;
ENGINE_REGISTRY[i].enabled = 1;
}
} else if (token[0] == '-' && token[1] != '\0') {
generated by cgit v1.3 (git 2.53.0) at 2026-09-07 12:17:50 +0000