aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/Config.c64
-rw-r--r--src/Config.h2
-rw-r--r--src/Main.c18
-rw-r--r--src/Routes/Images.c85
-rw-r--r--src/Routes/Search.c457
-rw-r--r--src/Routes/Settings.c16
-rw-r--r--src/Routes/SettingsSave.c30
-rw-r--r--src/Scraping/Scraping.c131
-rw-r--r--src/Scraping/Scraping.h2
-rw-r--r--src/Scraping/ScrapingHttp.c13
-rw-r--r--src/Scraping/ScrapingParsers.c116
-rw-r--r--src/Utility/Utility.c199
-rw-r--r--src/Utility/Utility.h22
13 files changed, 652 insertions, 503 deletions
diff --git a/src/Config.c b/src/Config.c
index 9883d45..3d2790b 100644
--- a/src/Config.c
+++ b/src/Config.c
@@ -3,6 +3,17 @@
#include <stdlib.h>
#include <string.h>
+static char *trim(char *value) {
+ while (*value == ' ' || *value == '\t')
+ value++;
+
+ char *end = value + strlen(value);
+ while (end > value && (end[-1] == ' ' || end[-1] == '\t'))
+ end--;
+ *end = '\0';
+ return value;
+}
+
int load_config(const char *filename, Config *config) {
FILE *file = fopen(filename, "r");
if (!file) {
@@ -33,51 +44,31 @@ int load_config(const char *filename, Config *config) {
char *delimiter = strchr(line, '=');
if (delimiter) {
*delimiter = '\0';
- char *key = line;
- char *value = delimiter + 1;
-
- while (*key == ' ' || *key == '\t')
- key++;
- while (*value == ' ' || *value == '\t')
- value++;
+ char *key = trim(line);
+ char *value = trim(delimiter + 1);
- char *key_end = key + strlen(key) - 1;
- while (key_end > key && (*key_end == ' ' || *key_end == '\t')) {
- *key_end = '\0';
- key_end--;
- }
-
- char *value_end = value + strlen(value) - 1;
- while (value_end > value && (*value_end == ' ' || *value_end == '\t' ||
- *value_end == '"' || *value_end == '\'')) {
- *value_end = '\0';
- value_end--;
- }
-
- while (*value == ' ' || *value == '\t')
- value++;
+ char *value_end = value + strlen(value);
+ while (value_end > value &&
+ (value_end[-1] == '"' || value_end[-1] == '\''))
+ *--value_end = '\0';
while (*value == '"' || *value == '\'')
value++;
if (strcmp(section, "server") == 0) {
if (strcmp(key, "host") == 0) {
- strncpy(config->host, value, sizeof(config->host) - 1);
- config->host[sizeof(config->host) - 1] = '\0';
+ snprintf(config->host, sizeof(config->host), "%s", value);
} else if (strcmp(key, "port") == 0) {
config->port = atoi(value);
} else if (strcmp(key, "locale") == 0) {
- strncpy(config->default_locale, value,
- sizeof(config->default_locale) - 1);
- config->default_locale[sizeof(config->default_locale) - 1] = '\0';
+ snprintf(config->default_locale, sizeof(config->default_locale), "%s",
+ value);
}
} else if (strcmp(section, "proxy") == 0) {
if (strcmp(key, "proxy") == 0) {
- strncpy(config->proxy, value, sizeof(config->proxy) - 1);
- config->proxy[sizeof(config->proxy) - 1] = '\0';
+ snprintf(config->proxy, sizeof(config->proxy), "%s", value);
} else if (strcmp(key, "list_file") == 0) {
- strncpy(config->proxy_list_file, value,
- sizeof(config->proxy_list_file) - 1);
- config->proxy_list_file[sizeof(config->proxy_list_file) - 1] = '\0';
+ snprintf(config->proxy_list_file, sizeof(config->proxy_list_file),
+ "%s", value);
} else if (strcmp(key, "max_retries") == 0) {
config->max_proxy_retries = atoi(value);
} else if (strcmp(key, "randomize_username") == 0) {
@@ -87,8 +78,7 @@ int load_config(const char *filename, Config *config) {
}
} else if (strcmp(section, "cache") == 0) {
if (strcmp(key, "dir") == 0) {
- strncpy(config->cache_dir, value, sizeof(config->cache_dir) - 1);
- config->cache_dir[sizeof(config->cache_dir) - 1] = '\0';
+ snprintf(config->cache_dir, sizeof(config->cache_dir), "%s", value);
} else if (strcmp(key, "ttl_search") == 0) {
config->cache_ttl_search = atoi(value);
} else if (strcmp(key, "ttl_infobox") == 0) {
@@ -98,8 +88,10 @@ int load_config(const char *filename, Config *config) {
}
} else if (strcmp(section, "engines") == 0) {
if (strcmp(key, "engines") == 0) {
- strncpy(config->engines, value, sizeof(config->engines) - 1);
- config->engines[sizeof(config->engines) - 1] = '\0';
+ snprintf(config->engines, sizeof(config->engines), "%s", value);
+ } else if (strcmp(key, "yacy_instance") == 0) {
+ snprintf(config->yacy_instance, sizeof(config->yacy_instance), "%s",
+ value);
}
} else if (strcmp(section, "rate_limit") == 0) {
if (strcmp(key, "search_requests") == 0) {
diff --git a/src/Config.h b/src/Config.h
index 25bd978..f0bdf4b 100644
--- a/src/Config.h
+++ b/src/Config.h
@@ -8,6 +8,7 @@
#define DEFAULT_CACHE_TTL_INFOBOX 86400
#define DEFAULT_CACHE_TTL_IMAGE 604800
#define DEFAULT_MAX_PROXY_RETRIES 3
+#define DEFAULT_YACY_INSTANCE "http://127.0.0.1:8090"
#define BUFFER_SIZE_SMALL 256
#define BUFFER_SIZE_MEDIUM 512
@@ -46,6 +47,7 @@ typedef struct {
int cache_ttl_infobox;
int cache_ttl_image;
char engines[512];
+ char yacy_instance[512];
int rate_limit_search_requests;
int rate_limit_search_interval;
int rate_limit_images_requests;
diff --git a/src/Main.c b/src/Main.c
index d7ff185..868244c 100644
--- a/src/Main.c
+++ b/src/Main.c
@@ -78,6 +78,7 @@ int main() {
.cache_ttl_infobox = DEFAULT_CACHE_TTL_INFOBOX,
.cache_ttl_image = DEFAULT_CACHE_TTL_IMAGE,
.engines = "",
+ .yacy_instance = "",
.rate_limit_search_requests = 0,
.rate_limit_search_interval = 0,
.rate_limit_images_requests = 0,
@@ -101,6 +102,23 @@ int main() {
}
apply_engines_config(cfg.engines);
+ configure_yacy_engine(cfg.yacy_instance);
+
+ if (cfg.yacy_instance[0] != '\0') {
+ int yacy_enabled = 0;
+ for (int i = 0; i < ENGINE_COUNT; i++) {
+ if (strcmp(ENGINE_REGISTRY[i].id, "yacy") == 0) {
+ yacy_enabled = ENGINE_REGISTRY[i].enabled;
+ break;
+ }
+ }
+ if (!yacy_enabled) {
+ fprintf(
+ stderr,
+ "[INFO] YaCy instance configured but the yacy engine is not "
+ "enabled (add it to the engines list, e.g. engines=\"*,yacy\")\n");
+ }
+ }
if (cache_init(cfg.cache_dir) != 0) {
fprintf(stderr,
diff --git a/src/Routes/Images.c b/src/Routes/Images.c
index 40ab88f..1ab875e 100644
--- a/src/Routes/Images.c
+++ b/src/Routes/Images.c
@@ -76,14 +76,12 @@ int images_handler(UrlParams *params) {
if (!error_images_msg)
error_images_msg = "Error fetching images";
- char ***pager_matrix = NULL;
- int *pager_inner_counts = NULL;
- int pager_count =
- build_pagination(page, images_href_builder, (void *)raw_query,
- &pager_matrix, &pager_inner_counts);
- if (pager_count > 0) {
- context_set_array_of_arrays(&ctx, "pagination_links", pager_matrix,
- pager_count, pager_inner_counts);
+ StringMatrix pagination;
+ string_matrix_build_pagination(page, images_href_builder, (void *)raw_query,
+ &pagination);
+ if (pagination.count > 0) {
+ context_set_array_of_arrays(&ctx, "pagination_links", pagination.rows,
+ pagination.count, pagination.field_counts);
}
char *display_query = url_decode_query(raw_query);
@@ -91,8 +89,9 @@ int images_handler(UrlParams *params) {
if (!raw_query || strlen(raw_query) == 0) {
send_redirect("/");
- if (display_query)
- free(display_query);
+ string_matrix_free(&pagination);
+ free(display_query);
+ free(locale);
free_context(&ctx);
return -1;
}
@@ -128,7 +127,9 @@ int images_handler(UrlParams *params) {
snprintf(response, sizeof(response), "<h1>%s</h1>", rate_limit_msg);
send_response(response);
free(request_cache_key);
+ string_matrix_free(&pagination);
free(display_query);
+ free(locale);
free_context(&ctx);
return -1;
}
@@ -147,37 +148,37 @@ int images_handler(UrlParams *params) {
snprintf(error_html, sizeof(error_html), "<h1>%s</h1>", error_images_msg);
send_response(error_html);
free(request_cache_key);
+ string_matrix_free(&pagination);
free(display_query);
+ free(locale);
free_context(&ctx);
return -1;
}
- char ***image_matrix = malloc(sizeof(char **) * result_count);
- int *inner_counts = malloc(sizeof(int) * result_count);
-
- if (!image_matrix || !inner_counts) {
- if (image_matrix)
- free(image_matrix);
- if (inner_counts)
- free(inner_counts);
- free_image_results(results, result_count);
- free(request_cache_key);
- free(display_query);
- free_context(&ctx);
- return -1;
- }
+ StringMatrix images;
+ string_matrix_init(&images);
for (int i = 0; i < result_count; i++) {
- image_matrix[i] = malloc(sizeof(char *) * IMAGE_RESULT_FIELDS);
- image_matrix[i][0] = strdup(results[i].thumbnail_url);
- image_matrix[i][1] = strdup(results[i].title);
- image_matrix[i][2] = strdup(results[i].page_url);
- image_matrix[i][3] = strdup(results[i].full_url);
- inner_counts[i] = IMAGE_RESULT_FIELDS;
+ const char *values[IMAGE_RESULT_FIELDS] = {
+ results[i].thumbnail_url,
+ results[i].title,
+ results[i].page_url,
+ results[i].full_url,
+ };
+ if (string_matrix_append(&images, values, IMAGE_RESULT_FIELDS) != 0) {
+ string_matrix_free(&images);
+ string_matrix_free(&pagination);
+ free_image_results(results, result_count);
+ free(request_cache_key);
+ free(display_query);
+ free(locale);
+ free_context(&ctx);
+ return -1;
+ }
}
- context_set_array_of_arrays(&ctx, "images", image_matrix, result_count,
- inner_counts);
+ context_set_array_of_arrays(&ctx, "images", images.rows, images.count,
+ images.field_counts);
char *rendered = render_template("images.html", &ctx);
if (rendered) {
@@ -187,27 +188,13 @@ int images_handler(UrlParams *params) {
send_response("<h1>Error rendering image results</h1>");
}
- for (int i = 0; i < result_count; i++) {
- for (int j = 0; j < IMAGE_RESULT_FIELDS; j++)
- free(image_matrix[i][j]);
- free(image_matrix[i]);
- }
- free(image_matrix);
- free(inner_counts);
-
- if (pager_count > 0) {
- for (int i = 0; i < pager_count; i++) {
- for (int j = 0; j < LINK_FIELD_COUNT; j++)
- free(pager_matrix[i][j]);
- free(pager_matrix[i]);
- }
- free(pager_matrix);
- free(pager_inner_counts);
- }
+ string_matrix_free(&images);
+ string_matrix_free(&pagination);
free_image_results(results, result_count);
free(request_cache_key);
free(display_query);
+ free(locale);
free_context(&ctx);
return 0;
diff --git a/src/Routes/Search.c b/src/Routes/Search.c
index b580c10..7fcdac8 100644
--- a/src/Routes/Search.c
+++ b/src/Routes/Search.c
@@ -140,29 +140,31 @@ static InfoBox fetch_currency_wrapper(char *query) {
return fetch_currency_data(query);
}
char *get_base_url(const char *input) {
- if (!input) return NULL;
+ if (!input)
+ return NULL;
- const char *start = input;
+ const char *start = input;
- const char *protocol_pos = strstr(input, "://");
- if (protocol_pos) {
- start = protocol_pos + 3;
- }
+ const char *protocol_pos = strstr(input, "://");
+ if (protocol_pos) {
+ start = protocol_pos + 3;
+ }
- const char *end = start;
- while (*end && *end != '/' && *end != '?' && *end != '#') {
- end++;
- }
+ const char *end = start;
+ while (*end && *end != '/' && *end != '?' && *end != '#') {
+ end++;
+ }
- size_t len = end - start;
+ size_t len = end - start;
- char *domain = (char *)malloc(len + 1);
- if (!domain) return NULL;
+ char *domain = (char *)malloc(len + 1);
+ if (!domain)
+ return NULL;
- strncpy(domain, start, len);
- domain[len] = '\0';
+ strncpy(domain, start, len);
+ domain[len] = '\0';
- return domain;
+ return domain;
}
static int is_calculator_query(const char *query) {
if (!query)
@@ -249,88 +251,65 @@ static void *infobox_thread_func(void *arg) {
return NULL;
}
-static int add_infobox_to_collection(InfoBox *infobox, char ****collection,
- int **inner_counts, int current_count) {
- *collection =
- (char ***)realloc(*collection, sizeof(char **) * (current_count + 1));
- *inner_counts =
- (int *)realloc(*inner_counts, sizeof(int) * (current_count + 1));
-
- (*collection)[current_count] =
- (char **)malloc(sizeof(char *) * INFOBOX_FIELD_COUNT);
- (*collection)[current_count][0] =
- infobox->title ? strdup(infobox->title) : NULL;
- (*collection)[current_count][1] =
- infobox->thumbnail_url ? strdup(infobox->thumbnail_url) : NULL;
- (*collection)[current_count][2] =
- infobox->extract ? strdup(infobox->extract) : NULL;
- (*collection)[current_count][3] = infobox->url ? strdup(infobox->url) : NULL;
- (*collection)[current_count][4] = infobox->url ? strdup(infobox->url) : NULL;
- (*inner_counts)[current_count] = INFOBOX_FIELD_COUNT;
-
- return current_count + 1;
+static int add_infobox_to_collection(InfoBox *infobox,
+ StringMatrix *collection) {
+ const char *values[INFOBOX_FIELD_COUNT] = {
+ infobox->title, infobox->thumbnail_url, infobox->extract,
+ infobox->url, infobox->url,
+ };
+ return string_matrix_append(collection, values, INFOBOX_FIELD_COUNT);
}
static int add_warning_to_collection(const char *engine_name,
const char *warning_message,
- char ****collection, int **inner_counts,
- int current_count) {
- char ***new_collection =
- (char ***)malloc(sizeof(char **) * (current_count + 1));
- int *new_inner_counts =
- (int *)malloc(sizeof(int) * (current_count + 1));
+ StringMatrix *collection) {
+ const char *values[] = {engine_name, warning_message};
+ return string_matrix_append(collection, values, 2);
+}
- if (!new_collection || !new_inner_counts) {
- free(new_collection);
- free(new_inner_counts);
- return current_count;
- }
+static void free_user_engine_list(char **user_engines, int user_engine_count) {
+ for (int i = 0; i < user_engine_count; i++)
+ free(user_engines[i]);
+ free(user_engines);
+}
- if (*collection && current_count > 0) {
- memcpy(new_collection, *collection, sizeof(char **) * current_count);
- }
- if (*inner_counts && current_count > 0) {
- memcpy(new_inner_counts, *inner_counts, sizeof(int) * current_count);
+static void free_search_results(SearchResult **all_results, ScrapeJob *jobs,
+ int job_count) {
+ for (int i = 0; i < job_count; i++) {
+ for (int j = 0; all_results[i] && j < jobs[i].results_count; j++) {
+ free(all_results[i][j].url);
+ free(all_results[i][j].title);
+ free(all_results[i][j].snippet);
+ }
+ free(all_results[i]);
}
+}
- free(*collection);
- free(*inner_counts);
-
- *collection = new_collection;
- *inner_counts = new_inner_counts;
-
- (*collection)[current_count] = (char **)malloc(sizeof(char *) * 2);
- if (!(*collection)[current_count])
- return current_count;
-
- (*collection)[current_count][0] = strdup(engine_name ? engine_name : "");
- (*collection)[current_count][1] =
- strdup(warning_message ? warning_message : "");
-
- if (!(*collection)[current_count][0] || !(*collection)[current_count][1]) {
- free((*collection)[current_count][0]);
- free((*collection)[current_count][1]);
- free((*collection)[current_count]);
- return current_count;
+static void free_infobox_results(InfoBoxThreadData *data) {
+ for (int i = 0; i < HANDLER_COUNT; i++) {
+ free_infobox(&data[i].result);
}
-
- (*inner_counts)[current_count] = 2;
- return current_count + 1;
}
-static const char *warning_message_for_job(const ScrapeJob *job, const char *locale) {
+static const char *warning_message_for_job(const ScrapeJob *job,
+ const char *locale) {
switch (job->status) {
case SCRAPE_STATUS_FETCH_ERROR: {
const char *msg = beaker_get_locale_value(locale, "warning_fetch_error");
- return msg ? msg : "request failed before OmniSearch could read search results.";
+ return msg ? msg
+ : "request failed before OmniSearch could read search results.";
}
case SCRAPE_STATUS_PARSE_MISMATCH: {
const char *msg = beaker_get_locale_value(locale, "warning_parse_mismatch");
- return msg ? msg : "returned search results in a format OmniSearch could not parse.";
+ return msg ? msg
+ : "returned search results in a format OmniSearch could not "
+ "parse.";
}
case SCRAPE_STATUS_BLOCKED: {
const char *msg = beaker_get_locale_value(locale, "warning_blocked");
- return msg ? msg : "returned a captcha or another blocking page instead of search results.";
+ return msg ? msg
+ : "returned a captcha or another blocking page instead of "
+ "search results.";
}
default:
return NULL;
@@ -384,8 +363,8 @@ static int engine_allowed_for_user(const SearchEngine *eng, char **user_ids,
static char *build_search_href(const char *query, const char *engine_id,
int page) {
const char *safe_query = query ? query : "";
- int use_engine = engine_id && engine_id[0] != '\0' &&
- !engine_id_matches(engine_id, "all");
+ int use_engine =
+ engine_id && engine_id[0] != '\0' && !engine_id_matches(engine_id, "all");
size_t needed = strlen("/search?q=") + strlen(safe_query) + 1;
if (use_engine)
@@ -437,13 +416,18 @@ int results_handler(UrlParams *params) {
extern Config global_config;
TemplateContext ctx = new_context();
char *raw_query = "";
- const char *selected_engine_id = "all";
+ char selected_engine_id[32];
+ char *default_engine = get_default_search_engine();
+ snprintf(selected_engine_id, sizeof(selected_engine_id), "%s",
+ default_engine);
+ free(default_engine);
int page = 1;
int btnI = 0;
char **user_engines = NULL;
int user_engine_count = 0;
- int has_user_pref = (get_user_engines(&user_engines, &user_engine_count) == 0);
+ int has_user_pref =
+ (get_user_engines(&user_engines, &user_engine_count) == 0);
if (params) {
for (int i = 0; i < params->count; i++) {
@@ -454,7 +438,8 @@ int results_handler(UrlParams *params) {
if (parsed > 1)
page = parsed;
} else if (strcmp(params->params[i].key, "engine") == 0) {
- selected_engine_id = params->params[i].value;
+ snprintf(selected_engine_id, sizeof(selected_engine_id), "%s",
+ params->params[i].value);
} else if (strcmp(params->params[i].key, "btnI") == 0) {
btnI = atoi(params->params[i].value);
}
@@ -471,9 +456,11 @@ int results_handler(UrlParams *params) {
beaker_set_locale(&ctx, locale);
const char *rate_limit_msg = beaker_get_locale_value(locale, "rate_limit");
- if (!rate_limit_msg) rate_limit_msg = "Slow down! Too many searches from you!";
+ if (!rate_limit_msg)
+ rate_limit_msg = "Slow down! Too many searches from you!";
const char *no_results_msg = beaker_get_locale_value(locale, "no_results");
- if (!no_results_msg) no_results_msg = "No results found";
+ if (!no_results_msg)
+ no_results_msg = "No results found";
char page_str[16];
snprintf(page_str, sizeof(page_str), "%d", page);
@@ -491,18 +478,24 @@ int results_handler(UrlParams *params) {
if (!raw_query || strlen(raw_query) == 0) {
send_redirect("/");
- if (has_user_pref) {
- for (int i = 0; i < user_engine_count; i++)
- free(user_engines[i]);
- free(user_engines);
- }
+ free(locale);
+ free_user_engine_list(user_engines, user_engine_count);
free_context(&ctx);
return -1;
}
const SearchEngine *selected_engine = find_enabled_engine(selected_engine_id);
- if (!selected_engine)
- selected_engine_id = "all";
+ if (selected_engine &&
+ !engine_allowed_for_user(selected_engine, user_engines,
+ user_engine_count, has_user_pref)) {
+ selected_engine = NULL;
+ }
+ if (selected_engine) {
+ snprintf(selected_engine_id, sizeof(selected_engine_id), "%s",
+ selected_engine->id);
+ } else {
+ snprintf(selected_engine_id, sizeof(selected_engine_id), "all");
+ }
context_set(&ctx, "selected_engine", selected_engine_id);
char *search_href = build_search_href(raw_query, selected_engine_id, 1);
@@ -528,13 +521,6 @@ int results_handler(UrlParams *params) {
infobox_data[i].result = (InfoBox){NULL};
}
- if (page == 1) {
- for (int i = 0; i < HANDLER_COUNT; i++) {
- pthread_create(&infobox_threads[i], NULL, infobox_thread_func,
- &infobox_data[i]);
- }
- }
-
ScrapeJob jobs[ENGINE_COUNT];
SearchResult *all_results[ENGINE_COUNT];
@@ -592,11 +578,8 @@ int results_handler(UrlParams *params) {
snprintf(response, sizeof(response), "<h1>%s</h1>", rate_limit_msg);
send_response(response);
free(request_cache_key);
- if (has_user_pref) {
- for (int i = 0; i < user_engine_count; i++)
- free(user_engines[i]);
- free(user_engines);
- }
+ free(locale);
+ free_user_engine_list(user_engines, user_engine_count);
free_context(&ctx);
return -1;
}
@@ -615,15 +598,14 @@ int results_handler(UrlParams *params) {
}
if (filter_engine_count > 1) {
- char ***filter_matrix = NULL;
- int *filter_inner_counts = NULL;
- int filter_count = 0;
+ StringMatrix filter_matrix;
+ string_matrix_init(&filter_matrix);
char *all_href = build_search_href(raw_query, "all", 1);
-
- filter_count = add_link_to_collection(
+ const char *all_values[LINK_FIELD_COUNT] = {
all_href, "All",
- selected_engine ? "engine-filter" : "engine-filter active",
- &filter_matrix, &filter_inner_counts, filter_count);
+ selected_engine ? "engine-filter" : "engine-filter active"};
+
+ string_matrix_append(&filter_matrix, all_values, LINK_FIELD_COUNT);
free(all_href);
for (int i = 0; i < ENGINE_COUNT; i++) {
@@ -639,22 +621,24 @@ int results_handler(UrlParams *params) {
? "engine-filter active"
: "engine-filter";
- filter_count = add_link_to_collection(filter_href, ENGINE_REGISTRY[i].name,
- filter_class, &filter_matrix,
- &filter_inner_counts, filter_count);
+ const char *filter_values[LINK_FIELD_COUNT] = {
+ filter_href, ENGINE_REGISTRY[i].name, filter_class};
+ string_matrix_append(&filter_matrix, filter_values, LINK_FIELD_COUNT);
free(filter_href);
}
- if (filter_count > 0) {
- context_set_array_of_arrays(&ctx, "engine_filters", filter_matrix,
- filter_count, filter_inner_counts);
- for (int i = 0; i < filter_count; i++) {
- for (int j = 0; j < LINK_FIELD_COUNT; j++)
- free(filter_matrix[i][j]);
- free(filter_matrix[i]);
- }
- free(filter_matrix);
- free(filter_inner_counts);
+ if (filter_matrix.count > 0) {
+ context_set_array_of_arrays(&ctx, "engine_filters", filter_matrix.rows,
+ filter_matrix.count,
+ filter_matrix.field_counts);
+ }
+ string_matrix_free(&filter_matrix);
+ }
+
+ if (page == 1) {
+ for (int i = 0; i < HANDLER_COUNT; i++) {
+ pthread_create(&infobox_threads[i], NULL, infobox_thread_func,
+ &infobox_data[i]);
}
}
@@ -672,27 +656,12 @@ int results_handler(UrlParams *params) {
for (int i = 0; i < engine_idx; i++) {
if (jobs[i].results_count > 0 && all_results[i][0].url) {
char *redirect_url = strdup(all_results[i][0].url);
- for (int j = 0; j < enabled_engine_count; j++) {
- for (int k = 0; k < jobs[j].results_count; k++) {
- free(all_results[j][k].url);
- free(all_results[j][k].title);
- free(all_results[j][k].snippet);
- }
- free(all_results[j]);
- }
- if (page == 1) {
- for (int j = 0; j < HANDLER_COUNT; j++) {
- if (infobox_data[j].success) {
- free_infobox(&infobox_data[j].result);
- }
- }
- }
+ free_search_results(all_results, jobs, engine_idx);
+ if (page == 1)
+ free_infobox_results(infobox_data);
free(request_cache_key);
- if (has_user_pref) {
- for (int i = 0; i < user_engine_count; i++)
- free(user_engines[i]);
- free(user_engines);
- }
+ free(locale);
+ free_user_engine_list(user_engines, user_engine_count);
free_context(&ctx);
if (redirect_url) {
send_redirect(redirect_url);
@@ -701,54 +670,37 @@ int results_handler(UrlParams *params) {
return 0;
}
}
- for (int i = 0; i < enabled_engine_count; i++) {
- free(all_results[i]);
- }
- if (page == 1) {
- for (int i = 0; i < HANDLER_COUNT; i++) {
- if (infobox_data[i].success) {
- free_infobox(&infobox_data[i].result);
- }
- }
- }
+ free_search_results(all_results, jobs, engine_idx);
+ if (page == 1)
+ free_infobox_results(infobox_data);
free(request_cache_key);
- if (has_user_pref) {
- for (int i = 0; i < user_engine_count; i++)
- free(user_engines[i]);
- free(user_engines);
- }
+ free(locale);
+ free_user_engine_list(user_engines, user_engine_count);
free_context(&ctx);
char no_results_html[128];
- snprintf(no_results_html, sizeof(no_results_html), "<h1>%s</h1>", no_results_msg);
+ snprintf(no_results_html, sizeof(no_results_html), "<h1>%s</h1>",
+ no_results_msg);
send_response(no_results_html);
return 0;
}
-
- char ***infobox_matrix = NULL;
- int *infobox_inner_counts = NULL;
- int infobox_count = 0;
+
+ StringMatrix infobox_matrix;
+ string_matrix_init(&infobox_matrix);
if (page == 1) {
for (int i = 0; i < HANDLER_COUNT; i++) {
if (infobox_data[i].success) {
- infobox_count =
- add_infobox_to_collection(&infobox_data[i].result, &infobox_matrix,
- &infobox_inner_counts, infobox_count);
+ add_infobox_to_collection(&infobox_data[i].result, &infobox_matrix);
}
}
}
- if (infobox_count > 0) {
- context_set_array_of_arrays(&ctx, "infoboxes", infobox_matrix,
- infobox_count, infobox_inner_counts);
- for (int i = 0; i < infobox_count; i++) {
- for (int j = 0; j < INFOBOX_FIELD_COUNT; j++)
- free(infobox_matrix[i][j]);
- free(infobox_matrix[i]);
- }
- free(infobox_matrix);
- free(infobox_inner_counts);
+ if (infobox_matrix.count > 0) {
+ context_set_array_of_arrays(&ctx, "infoboxes", infobox_matrix.rows,
+ infobox_matrix.count,
+ infobox_matrix.field_counts);
}
+ string_matrix_free(&infobox_matrix);
int warning_count = 0;
for (int i = 0; i < enabled_engine_count; i++) {
@@ -757,35 +709,25 @@ int results_handler(UrlParams *params) {
}
if (warning_count > 0) {
- char ***warning_matrix = NULL;
- int *warning_inner_counts = NULL;
- int warning_index = 0;
+ StringMatrix warning_matrix;
+ string_matrix_init(&warning_matrix);
for (int i = 0; i < enabled_engine_count; i++) {
const char *warning_message = warning_message_for_job(&jobs[i], locale);
if (!warning_message)
continue;
- warning_index = add_warning_to_collection(
- jobs[i].engine->name, warning_message, &warning_matrix,
- &warning_inner_counts, warning_index);
+ add_warning_to_collection(jobs[i].engine->name, warning_message,
+ &warning_matrix);
}
- if (warning_index > 0) {
- context_set_array_of_arrays(&ctx, "engine_warnings", warning_matrix,
- warning_index, warning_inner_counts);
+ if (warning_matrix.count > 0) {
+ context_set_array_of_arrays(&ctx, "engine_warnings", warning_matrix.rows,
+ warning_matrix.count,
+ warning_matrix.field_counts);
}
- if (warning_matrix) {
- for (int i = 0; i < warning_index; i++) {
- free(warning_matrix[i][0]);
- free(warning_matrix[i][1]);
- free(warning_matrix[i]);
- }
- free(warning_matrix);
- }
- if (warning_inner_counts)
- free(warning_inner_counts);
+ string_matrix_free(&warning_matrix);
}
int total_results = 0;
@@ -794,33 +736,8 @@ int results_handler(UrlParams *params) {
}
if (total_results > 0) {
- char ***results_matrix = (char ***)malloc(sizeof(char **) * total_results);
- int *results_inner_counts = (int *)malloc(sizeof(int) * total_results);
- if (!results_matrix || !results_inner_counts) {
- char *html = render_template("results.html", &ctx);
- if (html) {
- send_response(html);
- free(html);
- }
- for (int i = 0; i < enabled_engine_count; i++)
- free(all_results[i]);
- if (page == 1) {
- for (int i = 0; i < HANDLER_COUNT; i++) {
- if (infobox_data[i].success) {
- free_infobox(&infobox_data[i].result);
- }
- }
- }
- free(request_cache_key);
- if (has_user_pref) {
- for (int i = 0; i < user_engine_count; i++)
- free(user_engines[i]);
- free(user_engines);
- }
- free_context(&ctx);
- return 0;
- }
- int unique_count = 0;
+ StringMatrix results_matrix;
+ string_matrix_init(&results_matrix);
UrlHashTable url_table;
url_hash_init(&url_table);
@@ -837,61 +754,45 @@ int results_handler(UrlParams *params) {
url_hash_insert(&url_table, display_url);
- results_matrix[unique_count] =
- (char **)malloc(sizeof(char *) * RESULT_FIELD_COUNT);
- if (!results_matrix[unique_count]) {
- free(all_results[i][j].url);
- free(all_results[i][j].title);
- free(all_results[i][j].snippet);
- continue;
- }
char *pretty_url = pretty_display_url(display_url);
char *base_url = get_base_url(display_url);
-
- results_matrix[unique_count][0] = strdup(display_url);
- results_matrix[unique_count][1] = strdup(pretty_url);
- results_matrix[unique_count][2] = all_results[i][j].title
- ? strdup(all_results[i][j].title)
- : strdup("Untitled");
- results_matrix[unique_count][3] =
- all_results[i][j].snippet ? strdup(all_results[i][j].snippet)
- : strdup("");
- results_matrix[unique_count][4] = strdup(base_url ? base_url : "");
- results_matrix[unique_count][5] = strdup("");
-
- results_inner_counts[unique_count] = RESULT_FIELD_COUNT;
+ const char *values[RESULT_FIELD_COUNT] = {
+ display_url,
+ pretty_url,
+ all_results[i][j].title ? all_results[i][j].title : "Untitled",
+ all_results[i][j].snippet ? all_results[i][j].snippet : "",
+ base_url,
+ "",
+ };
+ int append_result =
+ string_matrix_append(&results_matrix, values, RESULT_FIELD_COUNT);
free(pretty_url);
free(base_url);
free(all_results[i][j].url);
free(all_results[i][j].title);
free(all_results[i][j].snippet);
-
- unique_count++;
+ if (append_result != 0) {
+ continue;
+ }
}
free(all_results[i]);
}
- context_set_array_of_arrays(&ctx, "results", results_matrix, unique_count,
- results_inner_counts);
+ context_set_array_of_arrays(&ctx, "results", results_matrix.rows,
+ results_matrix.count,
+ results_matrix.field_counts);
- char ***pager_matrix = NULL;
- int *pager_inner_counts = NULL;
- SearchHrefData href_data = { .query = raw_query, .engine_id = selected_engine_id };
- int pager_count = build_pagination(page, search_href_builder,
- &href_data, &pager_matrix,
- &pager_inner_counts);
+ StringMatrix pager_matrix;
+ SearchHrefData href_data = {.query = raw_query,
+ .engine_id = selected_engine_id};
+ int pagination_result = string_matrix_build_pagination(
+ page, search_href_builder, &href_data, &pager_matrix);
- if (pager_count > 0) {
- context_set_array_of_arrays(&ctx, "pagination_links", pager_matrix,
- pager_count, pager_inner_counts);
- for (int i = 0; i < pager_count; i++) {
- for (int j = 0; j < LINK_FIELD_COUNT; j++)
- free(pager_matrix[i][j]);
- free(pager_matrix[i]);
- }
- free(pager_matrix);
- free(pager_inner_counts);
+ if (pagination_result == 0 && pager_matrix.count > 0) {
+ context_set_array_of_arrays(&ctx, "pagination_links", pager_matrix.rows,
+ pager_matrix.count,
+ pager_matrix.field_counts);
}
char *html = render_template("results.html", &ctx);
@@ -900,13 +801,8 @@ int results_handler(UrlParams *params) {
free(html);
}
- for (int i = 0; i < unique_count; i++) {
- for (int j = 0; j < RESULT_FIELD_COUNT; j++)
- free(results_matrix[i][j]);
- free(results_matrix[i]);
- }
- free(results_matrix);
- free(results_inner_counts);
+ string_matrix_free(&pager_matrix);
+ string_matrix_free(&results_matrix);
url_hash_free(&url_table);
} else {
char *html = render_template("results.html", &ctx);
@@ -915,26 +811,15 @@ int results_handler(UrlParams *params) {
free(html);
}
- for (int i = 0; i < enabled_engine_count; i++) {
- free(all_results[i]);
- }
+ free_search_results(all_results, jobs, engine_idx);
}
free(request_cache_key);
- if (page == 1) {
- for (int i = 0; i < HANDLER_COUNT; i++) {
- if (infobox_data[i].success) {
- free_infobox(&infobox_data[i].result);
- }
- }
- }
+ if (page == 1)
+ free_infobox_results(infobox_data);
free(locale);
- if (has_user_pref) {
- for (int i = 0; i < user_engine_count; i++)
- free(user_engines[i]);
- free(user_engines);
- }
+ free_user_engine_list(user_engines, user_engine_count);
free_context(&ctx);
return 0;
diff --git a/src/Routes/Settings.c b/src/Routes/Settings.c
index eb3072b..05325e8 100644
--- a/src/Routes/Settings.c
+++ b/src/Routes/Settings.c
@@ -34,6 +34,14 @@ int settings_handler(UrlParams *params) {
int user_engine_count = 0;
int has_user_pref =
(get_user_engines(&user_engines, &user_engine_count) == 0);
+ char *default_engine = get_default_search_engine();
+
+ if (strcmp(default_engine, "all") != 0 && has_user_pref &&
+ !user_engines_contains(default_engine, user_engines,
+ user_engine_count)) {
+ free(default_engine);
+ default_engine = strdup("all");
+ }
int enabled_count = 0;
for (int i = 0; i < ENGINE_COUNT; i++) {
@@ -63,11 +71,13 @@ int settings_handler(UrlParams *params) {
}
}
- engine_data[idx] = malloc(sizeof(char *) * 3);
+ engine_data[idx] = malloc(sizeof(char *) * 4);
engine_data[idx][0] = (char *)ENGINE_REGISTRY[i].id;
engine_data[idx][1] = (char *)ENGINE_REGISTRY[i].name;
engine_data[idx][2] = is_selected ? "checked" : "";
- engine_inner[idx] = 3;
+ engine_data[idx][3] =
+ strcmp(default_engine, ENGINE_REGISTRY[i].id) == 0 ? "selected" : "";
+ engine_inner[idx] = 4;
idx++;
}
@@ -78,6 +88,7 @@ int settings_handler(UrlParams *params) {
context_set(&ctx, "query", query);
context_set(&ctx, "theme", theme);
context_set(&ctx, "locale", locale);
+ context_set(&ctx, "default_engine", default_engine);
context_set_array_of_arrays(&ctx, "locales", locale_data, locale_count,
inner_counts);
@@ -132,6 +143,7 @@ int settings_handler(UrlParams *params) {
send_response(rendered_html);
free(rendered_html);
+ free(default_engine);
free(theme);
free(locale);
free_context(&ctx);
diff --git a/src/Routes/SettingsSave.c b/src/Routes/SettingsSave.c
index cacff50..3cbb1d6 100644
--- a/src/Routes/SettingsSave.c
+++ b/src/Routes/SettingsSave.c
@@ -10,6 +10,8 @@ int settings_save_handler(UrlParams *params) {
const char *theme = "";
const char *locale = "";
const char *query = "";
+ const char *default_engine = "all";
+ int default_engine_present = 0;
int engines_present = 0;
char selected_ids[ENGINE_COUNT][32];
int selected_count = 0;
@@ -22,6 +24,9 @@ int settings_save_handler(UrlParams *params) {
locale = params->params[i].value;
} else if (strcmp(params->params[i].key, "q") == 0) {
query = params->params[i].value;
+ } else if (strcmp(params->params[i].key, "default_engine") == 0) {
+ default_engine = params->params[i].value;
+ default_engine_present = 1;
} else if (strcmp(params->params[i].key, "engines_present") == 0) {
engines_present = 1;
} else if (strncmp(params->params[i].key, "engine_", 7) == 0 &&
@@ -58,6 +63,31 @@ int settings_save_handler(UrlParams *params) {
false, false);
}
+ if (default_engine_present) {
+ const char *validated_default = "all";
+
+ for (int i = 0; i < ENGINE_COUNT; i++) {
+ if (!ENGINE_REGISTRY[i].enabled ||
+ strcmp(default_engine, ENGINE_REGISTRY[i].id) != 0)
+ continue;
+
+ int selected = !engines_present;
+ for (int j = 0; j < selected_count; j++) {
+ if (strcmp(selected_ids[j], ENGINE_REGISTRY[i].id) == 0) {
+ selected = 1;
+ break;
+ }
+ }
+
+ if (selected)
+ validated_default = ENGINE_REGISTRY[i].id;
+ break;
+ }
+
+ set_cookie("default_engine", validated_default,
+ "Fri, 31 Dec 2038 23:59:59 GMT", "/", false, false);
+ }
+
char redirect_url[512];
snprintf(redirect_url, sizeof(redirect_url), "/settings?q=%s", query);
send_redirect(redirect_url);
diff --git a/src/Scraping/Scraping.c b/src/Scraping/Scraping.c
index b81c216..6d94450 100644
--- a/src/Scraping/Scraping.c
+++ b/src/Scraping/Scraping.c
@@ -1,9 +1,11 @@
#include "Scraping.h"
#include "../Cache/Cache.h"
#include "../Proxy/Proxy.h"
+#include "../Utility/XmlHelper.h"
#include "Config.h"
#include <curl/curl.h>
#include <libxml/HTMLparser.h>
+#include <libxml/parser.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -24,7 +26,8 @@ static int response_is_startpage_captcha(const ScrapeJob *job,
return response_contains(response, "<title>Startpage Captcha</title>") ||
response_contains(response, "Startpage Captcha") ||
- response_contains(response, "/static-pages-assets/page-data/captcha/") ||
+ response_contains(response,
+ "/static-pages-assets/page-data/captcha/") ||
response_contains(response, ">Startpage Blocked</title>");
}
@@ -39,7 +42,8 @@ static int response_looks_like_results_page(const ScrapeJob *job,
}
if (strcmp(job->engine->name, "Startpage") == 0) {
- return response_contains(response, "<title>Startpage Search Results</title>") ||
+ return response_contains(response,
+ "<title>Startpage Search Results</title>") ||
response_contains(response, "class=\"w-gl") ||
response_contains(response, "data-testid=\"gl-title-link\"");
}
@@ -55,6 +59,11 @@ static int response_looks_like_results_page(const ScrapeJob *job,
response_contains(response, "Mojeek Search");
}
+ if (strcmp(job->engine->name, "YaCy") == 0) {
+ return response_contains(response, "<item") ||
+ response_contains(response, "<rss");
+ }
+
return 0;
}
@@ -72,18 +81,24 @@ static void classify_job_response(ScrapeJob *job, const char *response,
return;
}
- xmlDocPtr doc = htmlReadMemory(response, response_size, NULL, NULL,
- HTML_PARSE_RECOVER | HTML_PARSE_NOERROR |
- HTML_PARSE_NOWARNING);
+ xmlDocPtr doc;
+ if (job->engine->is_xml) {
+ doc = xmlReadMemory(response, response_size, NULL, NULL,
+ XML_PARSE_RECOVER | XML_PARSE_NOERROR |
+ XML_PARSE_NOWARNING);
+ } else {
+ doc = htmlReadMemory(response, response_size, NULL, NULL,
+ HTML_PARSE_RECOVER | HTML_PARSE_NOERROR |
+ HTML_PARSE_NOWARNING);
+ }
if (!doc) {
job->status = SCRAPE_STATUS_FETCH_ERROR;
return;
}
- job->results_count =
- job->engine->parser(job->engine->name, doc, job->out_results,
- job->max_results);
+ job->results_count = job->engine->parser(job->engine->name, doc,
+ job->out_results, job->max_results);
xmlFreeDoc(doc);
if (job->results_count > 0) {
@@ -156,14 +171,26 @@ void parse_and_cache_response(ScrapeJob *job) {
}
}
-void cleanup_job_handle(ScrapeJob *job, CURL *handle) {
+static void cleanup_job_results(ScrapeJob *job) {
+ if (!job || !job->out_results)
+ return;
+
+ xml_result_free(*job->out_results, job->results_count);
+ *job->out_results = NULL;
+ job->results_count = 0;
+}
+
+static void cleanup_job_handle(ScrapeJob *job, CURL *handle) {
struct curl_slist *headers = NULL;
- curl_easy_getinfo(handle, CURLINFO_PRIVATE, &headers);
+ if (handle)
+ curl_easy_getinfo(handle, CURLINFO_PRIVATE, &headers);
if (headers)
curl_slist_free_all(headers);
free(job->response.memory);
job->response.memory = NULL;
+ job->response.size = 0;
+ job->response.capacity = 0;
}
void process_response(ScrapeJob *job, CURL *handle, CURLMsg *msg) {
@@ -180,19 +207,25 @@ void process_response(ScrapeJob *job, CURL *handle, CURLMsg *msg) {
}
int setup_job(ScrapeJob *job, CURLM *multi_handle) {
- if (job->handle)
+ if (job->handle) {
+ cleanup_job_handle(job, job->handle);
curl_easy_cleanup(job->handle);
- if (job->response.memory)
+ job->handle = NULL;
+ } else if (job->response.memory) {
free(job->response.memory);
+ job->response.memory = NULL;
+ job->response.size = 0;
+ job->response.capacity = 0;
+ }
- job->results_count = 0;
+ cleanup_job_results(job);
job->http_status = 0;
job->status = SCRAPE_STATUS_PENDING;
- if (check_cache_for_job(job)) {
- job->results_count = job->results_count > 0 ? job->results_count : 0;
+ if (check_cache_for_job(job))
return 0;
- }
+
+ cleanup_job_results(job);
char *encoded_query = curl_easy_escape(NULL, job->query, 0);
if (!encoded_query) {
@@ -200,12 +233,17 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) {
return -1;
}
- for (char *p = encoded_query + strlen(encoded_query) - 3; p >= encoded_query; p--) {
- if (p[0] == '%' && p[1] == '2' && p[2] == '0') {
- *p = '+';
- memmove(p + 1, p + 3, strlen(p + 3) + 1);
+ char *read = encoded_query;
+ char *write = encoded_query;
+ while (*read) {
+ if (read[0] == '%' && read[1] == '2' && read[2] == '0') {
+ *write++ = '+';
+ read += 3;
+ } else {
+ *write++ = *read++;
}
}
+ *write = '\0';
char *full_url =
build_search_url(job->engine->base_url, job->engine->page_param,
@@ -228,6 +266,14 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) {
job->response.memory = (char *)malloc(INITIAL_BUFFER_SIZE);
job->response.size = 0;
job->response.capacity = INITIAL_BUFFER_SIZE;
+ if (!job->response.memory) {
+ curl_easy_cleanup(job->handle);
+ job->handle = NULL;
+ job->response.capacity = 0;
+ free(full_url);
+ job->status = SCRAPE_STATUS_FETCH_ERROR;
+ return -1;
+ }
struct curl_slist *headers =
build_request_headers(job->engine->host_header, job->engine->referer);
@@ -236,10 +282,33 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) {
curl_easy_setopt(job->handle, CURLOPT_PRIVATE, headers);
free(full_url);
- curl_multi_add_handle(multi_handle, job->handle);
+ CURLMcode add_result = curl_multi_add_handle(multi_handle, job->handle);
+ if (add_result != CURLM_OK) {
+ cleanup_job_handle(job, job->handle);
+ curl_easy_cleanup(job->handle);
+ job->handle = NULL;
+ job->status = SCRAPE_STATUS_FETCH_ERROR;
+ return -1;
+ }
return 0;
}
+static void cleanup_unfinished_jobs(CURLM *multi_handle, ScrapeJob *jobs,
+ int num_jobs) {
+ for (int i = 0; i < num_jobs; i++) {
+ if (!jobs[i].handle)
+ continue;
+
+ curl_multi_remove_handle(multi_handle, jobs[i].handle);
+ cleanup_job_handle(&jobs[i], jobs[i].handle);
+ curl_easy_cleanup(jobs[i].handle);
+ jobs[i].handle = NULL;
+
+ if (jobs[i].status == SCRAPE_STATUS_PENDING)
+ jobs[i].status = SCRAPE_STATUS_FETCH_ERROR;
+ }
+}
+
int handle_responses(CURLM *multi_handle, ScrapeJob *jobs, int num_jobs) {
CURLMsg *msg;
int msgs_left;
@@ -279,34 +348,30 @@ int should_retry(ScrapeJob *jobs, int num_jobs) {
int scrape_engines_parallel(ScrapeJob *jobs, int num_jobs) {
int retries = 0;
-retry:
- ;
+retry:;
CURLM *multi_handle = curl_multi_init();
if (!multi_handle)
return -1;
for (int i = 0; i < num_jobs; i++) {
- if (setup_job(&jobs[i], multi_handle) != 0 && jobs[i].handle) {
- curl_multi_remove_handle(multi_handle, jobs[i].handle);
- curl_easy_cleanup(jobs[i].handle);
- jobs[i].handle = NULL;
- }
+ setup_job(&jobs[i], multi_handle);
}
http_delay();
int still_running = 0;
- curl_multi_perform(multi_handle, &still_running);
+ CURLMcode mc = curl_multi_perform(multi_handle, &still_running);
- do {
+ while (mc == CURLM_OK && still_running) {
int numfds = 0;
- CURLMcode mc = curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds);
+ mc = curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds);
if (mc != CURLM_OK)
break;
- curl_multi_perform(multi_handle, &still_running);
- } while (still_running);
+ mc = curl_multi_perform(multi_handle, &still_running);
+ }
handle_responses(multi_handle, jobs, num_jobs);
+ cleanup_unfinished_jobs(multi_handle, jobs, num_jobs);
curl_multi_cleanup(multi_handle);
if (retries < max_proxy_retries && should_retry(jobs, num_jobs)) {
diff --git a/src/Scraping/Scraping.h b/src/Scraping/Scraping.h
index be65e5a..14e6e55 100644
--- a/src/Scraping/Scraping.h
+++ b/src/Scraping/Scraping.h
@@ -26,6 +26,7 @@ typedef struct {
int page_base;
ParserFunc parser;
int enabled;
+ int is_xml;
} SearchEngine;
typedef struct {
@@ -59,6 +60,7 @@ typedef struct {
extern SearchEngine ENGINE_REGISTRY[];
extern const int ENGINE_COUNT;
void apply_engines_config(const char *engines_str);
+void configure_yacy_engine(const char *instance);
size_t write_memory_callback(void *contents, size_t size, size_t nmemb,
void *userp);
diff --git a/src/Scraping/ScrapingHttp.c b/src/Scraping/ScrapingHttp.c
index 1a6a292..89f4853 100644
--- a/src/Scraping/ScrapingHttp.c
+++ b/src/Scraping/ScrapingHttp.c
@@ -89,11 +89,14 @@ struct curl_slist *build_request_headers(const char *host_header,
struct curl_slist *headers = NULL;
char host_buf[BUFFER_SIZE_MEDIUM], ref_buf[BUFFER_SIZE_MEDIUM];
- snprintf(host_buf, sizeof(host_buf), "Host: %s", host_header);
- snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer);
-
- headers = curl_slist_append(headers, host_buf);
- headers = curl_slist_append(headers, ref_buf);
+ if (host_header && host_header[0] != '\0') {
+ snprintf(host_buf, sizeof(host_buf), "Host: %s", host_header);
+ headers = curl_slist_append(headers, host_buf);
+ }
+ if (referer && referer[0] != '\0') {
+ snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer);
+ headers = curl_slist_append(headers, ref_buf);
+ }
headers = curl_slist_append(
headers,
"Accept: "
diff --git a/src/Scraping/ScrapingParsers.c b/src/Scraping/ScrapingParsers.c
index 96aaded..2e5689b 100644
--- a/src/Scraping/ScrapingParsers.c
+++ b/src/Scraping/ScrapingParsers.c
@@ -249,8 +249,8 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
if (!ctx)
return 0;
- xmlXPathObjectPtr obj =
- xml_xpath_eval(ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]");
+ xmlXPathObjectPtr obj = xml_xpath_eval(
+ ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]");
if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) {
free_xpath_objects(ctx, obj);
@@ -268,18 +268,17 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
xmlNodePtr result_node = obj->nodesetval->nodeTab[i];
ctx->node = result_node;
- xmlXPathObjectPtr link_obj =
- xml_xpath_eval(ctx, ".//a[@class='title']");
+ xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, ".//a[@class='title']");
char *url =
(link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0)
? (char *)xmlGetProp(link_obj->nodesetval->nodeTab[0],
(xmlChar *)"href")
: NULL;
- char *title = (link_obj && link_obj->nodesetval &&
- link_obj->nodesetval->nodeNr > 0)
- ? xml_node_content(link_obj->nodesetval->nodeTab[0])
- : NULL;
+ char *title =
+ (link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0)
+ ? xml_node_content(link_obj->nodesetval->nodeTab[0])
+ : NULL;
xmlXPathObjectPtr snippet_obj = xml_xpath_eval(ctx, ".//p[@class='s']");
char *snippet_text =
@@ -310,6 +309,90 @@ static int parse_yahoo(const char *engine_name, xmlDocPtr doc,
static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
SearchResult **out_results, int max_results);
+static int parse_yacy(const char *engine_name, xmlDocPtr doc,
+ SearchResult **out_results, int max_results) {
+ (void)engine_name;
+ int found_count = 0;
+
+ xmlXPathContextPtr ctx = create_xpath_context(doc);
+ if (!ctx)
+ return 0;
+
+ xmlXPathObjectPtr obj = xml_xpath_eval(ctx, "//item");
+
+ if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) {
+ free_xpath_objects(ctx, obj);
+ return 0;
+ }
+
+ int num_items = obj->nodesetval->nodeNr;
+ *out_results = alloc_results_array(num_items, max_results);
+ if (!*out_results) {
+ free_xpath_objects(ctx, obj);
+ return 0;
+ }
+
+ for (int i = 0; i < num_items && found_count < max_results; i++) {
+ xmlNodePtr item_node = obj->nodesetval->nodeTab[i];
+ ctx->node = item_node;
+
+ xmlXPathObjectPtr title_obj = xml_xpath_eval(ctx, "./title");
+ char *title = (title_obj && title_obj->nodesetval &&
+ title_obj->nodesetval->nodeNr > 0)
+ ? xml_node_content(title_obj->nodesetval->nodeTab[0])
+ : NULL;
+
+ xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, "./link");
+ char *url =
+ (link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0)
+ ? xml_node_content(link_obj->nodesetval->nodeTab[0])
+ : NULL;
+
+ xmlXPathObjectPtr desc_obj = xml_xpath_eval(ctx, "./description");
+ char *snippet_text =
+ (desc_obj && desc_obj->nodesetval && desc_obj->nodesetval->nodeNr > 0)
+ ? xml_node_content(desc_obj->nodesetval->nodeTab[0])
+ : NULL;
+
+ if (url && title) {
+ assign_result(&(*out_results)[found_count], url, title, snippet_text, 0);
+ found_count++;
+ }
+
+ free_xml_node_list(title, url, snippet_text);
+ if (title_obj)
+ xmlXPathFreeObject(title_obj);
+ if (link_obj)
+ xmlXPathFreeObject(link_obj);
+ if (desc_obj)
+ xmlXPathFreeObject(desc_obj);
+ }
+
+ ctx->node = NULL;
+ free_xpath_objects(ctx, obj);
+ return found_count;
+}
+
+static char yacy_base_url[BUFFER_SIZE_LARGE];
+
+void configure_yacy_engine(const char *instance) {
+ if (!instance || instance[0] == '\0')
+ instance = DEFAULT_YACY_INSTANCE;
+
+ snprintf(yacy_base_url, sizeof(yacy_base_url),
+ "%s/"
+ "yacysearch.rss?resource=global&urlmaskfilter=.*&prefermaskfilter=&"
+ "nav=all&maximumRecords=%d&query=",
+ instance, MAX_RESULTS_PER_ENGINE);
+
+ for (int i = 0; i < ENGINE_COUNT; i++) {
+ if (strcmp(ENGINE_REGISTRY[i].id, "yacy") == 0) {
+ ENGINE_REGISTRY[i].base_url = yacy_base_url;
+ break;
+ }
+ }
+}
+
SearchEngine ENGINE_REGISTRY[] = {
{.id = "ddg",
.name = "DuckDuckGo Lite",
@@ -350,7 +433,18 @@ SearchEngine ENGINE_REGISTRY[] = {
.page_multiplier = 10,
.page_base = 1,
.parser = parse_mojeek,
- .enabled = 1}};
+ .enabled = 1},
+ {.id = "yacy",
+ .name = "YaCy",
+ .base_url = "",
+ .host_header = NULL,
+ .referer = NULL,
+ .page_param = "startRecord",
+ .page_multiplier = 10,
+ .page_base = 0,
+ .parser = parse_yacy,
+ .enabled = 0,
+ .is_xml = 1}};
const int ENGINE_COUNT = sizeof(ENGINE_REGISTRY) / sizeof(SearchEngine);
@@ -373,6 +467,8 @@ static int engine_id_compare(const char *engine_id, const char *config_id) {
void apply_engines_config(const char *engines_str) {
if (!engines_str || engines_str[0] == '\0') {
for (int i = 0; i < ENGINE_COUNT; i++) {
+ if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy"))
+ continue;
ENGINE_REGISTRY[i].enabled = 1;
}
return;
@@ -395,6 +491,8 @@ void apply_engines_config(const char *engines_str) {
if (strcmp(token, "*") == 0) {
for (int i = 0; i < ENGINE_COUNT; i++) {
+ if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy"))
+ continue;
ENGINE_REGISTRY[i].enabled = 1;
}
} else if (token[0] == '-' && token[1] != '\0') {
diff --git a/src/Utility/Utility.c b/src/Utility/Utility.c
index 1428722..35fc491 100644
--- a/src/Utility/Utility.c
+++ b/src/Utility/Utility.c
@@ -14,7 +14,6 @@ static int themes_initialized = 0;
void init_themes(const char *static_path) {
if (themes_initialized)
return;
- themes_initialized = 1;
char themes_dir[512];
snprintf(themes_dir, sizeof(themes_dir), "%s/themes", static_path);
@@ -26,20 +25,32 @@ void init_themes(const char *static_path) {
struct dirent *entry;
int capacity = 4;
themes_list = malloc(sizeof(char *) * capacity);
+ if (!themes_list) {
+ closedir(dir);
+ return;
+ }
themes_count = 0;
while ((entry = readdir(dir)) != NULL) {
size_t len = strlen(entry->d_name);
if (len > 4 && strcmp(entry->d_name + len - 4, ".css") == 0) {
if (themes_count >= capacity) {
- capacity *= 2;
- themes_list = realloc(themes_list, sizeof(char *) * capacity);
+ int new_capacity = capacity * 2;
+ char **new_list = realloc(themes_list, sizeof(char *) * new_capacity);
+ if (!new_list)
+ break;
+ themes_list = new_list;
+ capacity = new_capacity;
}
- themes_list[themes_count] = strndup(entry->d_name, len - 4);
+ char *theme = strndup(entry->d_name, len - 4);
+ if (!theme)
+ break;
+ themes_list[themes_count] = theme;
themes_count++;
}
}
closedir(dir);
+ themes_initialized = 1;
for (int i = 0; i < themes_count; i++) {
for (int j = i + 1; j < themes_count; j++) {
@@ -206,98 +217,130 @@ int user_engines_contains(const char *engine_id, char **ids, int count) {
return 0;
}
-int add_link_to_collection(const char *href, const char *label,
- const char *class_name, char ****collection,
- int **inner_counts, int current_count) {
- char ***old_collection = *collection;
- int *old_inner_counts = *inner_counts;
- char ***new_collection =
- (char ***)malloc(sizeof(char **) * (current_count + 1));
- int *new_inner_counts = (int *)malloc(sizeof(int) * (current_count + 1));
+char *get_default_search_engine(void) {
+ char *cookie = get_cookie("default_engine");
+ if (cookie && (strcmp(cookie, "all") == 0 || is_engine_id_enabled(cookie)))
+ return cookie;
- if (!new_collection || !new_inner_counts) {
- free(new_collection);
- free(new_inner_counts);
- return current_count;
- }
+ free(cookie);
+ return strdup("all");
+}
- if (*collection && current_count > 0) {
- memcpy(new_collection, *collection, sizeof(char **) * current_count);
- }
- if (*inner_counts && current_count > 0) {
- memcpy(new_inner_counts, *inner_counts, sizeof(int) * current_count);
+void free_string_matrix(char ***matrix, int *inner_counts, int row_count) {
+ if (matrix) {
+ for (int i = 0; i < row_count; i++) {
+ int field_count = inner_counts ? inner_counts[i] : 0;
+ for (int j = 0; j < field_count; j++)
+ free(matrix[i][j]);
+ free(matrix[i]);
+ }
}
+ free(matrix);
+ free(inner_counts);
+}
- *collection = new_collection;
- *inner_counts = new_inner_counts;
+void string_matrix_init(StringMatrix *matrix) {
+ if (matrix)
+ *matrix = (StringMatrix){0};
+}
- (*collection)[current_count] =
- (char **)malloc(sizeof(char *) * LINK_FIELD_COUNT);
- if (!(*collection)[current_count]) {
- *collection = old_collection;
- *inner_counts = old_inner_counts;
- free(new_collection);
- free(new_inner_counts);
- return current_count;
- }
+int string_matrix_append(StringMatrix *matrix, const char *const *values,
+ int field_count) {
+ if (!matrix || !values || field_count <= 0)
+ return -1;
+
+ char **row = calloc((size_t)field_count, sizeof(*row));
+ if (!row)
+ return -1;
- (*collection)[current_count][0] = strdup(href ? href : "");
- (*collection)[current_count][1] = strdup(label ? label : "");
- (*collection)[current_count][2] = strdup(class_name ? class_name : "");
+ for (int i = 0; i < field_count; i++) {
+ row[i] = strdup(values[i] ? values[i] : "");
+ if (!row[i]) {
+ for (int j = 0; j < i; j++)
+ free(row[j]);
+ free(row);
+ return -1;
+ }
+ }
- if (!(*collection)[current_count][0] || !(*collection)[current_count][1] ||
- !(*collection)[current_count][2]) {
- free((*collection)[current_count][0]);
- free((*collection)[current_count][1]);
- free((*collection)[current_count][2]);
- free((*collection)[current_count]);
- *collection = old_collection;
- *inner_counts = old_inner_counts;
- free(new_collection);
- free(new_inner_counts);
- return current_count;
+ if (matrix->count == matrix->capacity) {
+ int new_capacity = matrix->capacity ? matrix->capacity * 2 : 8;
+ char ***new_rows = malloc(sizeof(*new_rows) * (size_t)new_capacity);
+ int *new_counts = malloc(sizeof(*new_counts) * (size_t)new_capacity);
+ if (!new_rows || !new_counts) {
+ free(new_rows);
+ free(new_counts);
+ for (int i = 0; i < field_count; i++)
+ free(row[i]);
+ free(row);
+ return -1;
+ }
+ if (matrix->count > 0) {
+ memcpy(new_rows, matrix->rows, sizeof(*new_rows) * matrix->count);
+ memcpy(new_counts, matrix->field_counts,
+ sizeof(*new_counts) * matrix->count);
+ }
+ free(matrix->rows);
+ free(matrix->field_counts);
+ matrix->rows = new_rows;
+ matrix->field_counts = new_counts;
+ matrix->capacity = new_capacity;
}
- (*inner_counts)[current_count] = LINK_FIELD_COUNT;
+ matrix->rows[matrix->count] = row;
+ matrix->field_counts[matrix->count] = field_count;
+ matrix->count++;
+ return 0;
+}
- free(old_collection);
- free(old_inner_counts);
- return current_count + 1;
+void string_matrix_free(StringMatrix *matrix) {
+ if (!matrix)
+ return;
+ free_string_matrix(matrix->rows, matrix->field_counts, matrix->count);
+ *matrix = (StringMatrix){0};
}
-int build_pagination(int page, char *(*href_builder)(int page, void *data),
- void *data, char ****out_matrix, int **out_inner_counts) {
+int string_matrix_build_pagination(int page,
+ char *(*href_builder)(int page, void *data),
+ void *data, StringMatrix *out_matrix) {
enum { PAGER_WINDOW_SIZE = 5 };
+ if (!out_matrix || !href_builder)
+ return -1;
- *out_matrix = NULL;
- *out_inner_counts = NULL;
- int count = 0;
-
+ string_matrix_init(out_matrix);
int pager_start = page <= 3 ? 1 : page - 2;
int pager_end = pager_start + PAGER_WINDOW_SIZE - 1;
- if (page > 1) {
- char *href = href_builder(page - 1, data);
- count = add_link_to_collection(href, "←", "pagination-btn prev", out_matrix,
- out_inner_counts, count);
- free(href);
- }
+ for (int i = page > 1 ? pager_start - 1 : pager_start; i <= pager_end + 1;
+ i++) {
+ const char *label = NULL;
+ const char *class_name = "pagination-btn";
+ char page_label[16];
+ int target_page = i;
+
+ if (page > 1 && i == pager_start - 1) {
+ label = "←";
+ class_name = "pagination-btn prev";
+ target_page = page - 1;
+ } else if (i == pager_end + 1) {
+ label = "→";
+ class_name = "pagination-btn next";
+ target_page = page + 1;
+ } else {
+ snprintf(page_label, sizeof(page_label), "%d", i);
+ label = page_label;
+ if (i == page)
+ class_name = "pagination-btn pagination-current";
+ }
- for (int i = pager_start; i <= pager_end; i++) {
- char label[16];
- snprintf(label, sizeof(label), "%d", i);
- char *href = href_builder(i, data);
- count = add_link_to_collection(
- href, label,
- i == page ? "pagination-btn pagination-current" : "pagination-btn",
- out_matrix, out_inner_counts, count);
+ char *href = href_builder(target_page, data);
+ const char *values[LINK_FIELD_COUNT] = {href, label, class_name};
+ int result = string_matrix_append(out_matrix, values, LINK_FIELD_COUNT);
free(href);
+ if (result != 0) {
+ string_matrix_free(out_matrix);
+ return -1;
+ }
}
-
- char *href = href_builder(page + 1, data);
- count = add_link_to_collection(href, "→", "pagination-btn next", out_matrix,
- out_inner_counts, count);
- free(href);
-
- return count;
+ return 0;
}
diff --git a/src/Utility/Utility.h b/src/Utility/Utility.h
index 1e1de09..a194f21 100644
--- a/src/Utility/Utility.h
+++ b/src/Utility/Utility.h
@@ -13,6 +13,13 @@
#define LINK_FIELD_COUNT 3
+typedef struct {
+ char ***rows;
+ int *field_counts;
+ int count;
+ int capacity;
+} StringMatrix;
+
int hex_to_int(char c);
char *get_theme(const char *default_theme);
void init_themes(const char *static_path);
@@ -23,12 +30,17 @@ char *get_locale(const char *default_locale);
int is_engine_id_enabled(const char *engine_id);
int get_user_engines(char ***out_ids, int *out_count);
int user_engines_contains(const char *engine_id, char **ids, int count);
+char *get_default_search_engine(void);
+
+void free_string_matrix(char ***matrix, int *inner_counts, int row_count);
-int add_link_to_collection(const char *href, const char *label,
- const char *class_name, char ****collection,
- int **inner_counts, int current_count);
+void string_matrix_init(StringMatrix *matrix);
+int string_matrix_append(StringMatrix *matrix, const char *const *values,
+ int field_count);
+void string_matrix_free(StringMatrix *matrix);
-int build_pagination(int page, char *(*href_builder)(int page, void *data),
- void *data, char ****out_matrix, int **out_inner_counts);
+int string_matrix_build_pagination(int page,
+ char *(*href_builder)(int page, void *data),
+ void *data, StringMatrix *out_matrix);
#endif
generated by cgit v1.3 (git 2.53.0) at 2026-09-07 16:32:02 +0000