From 7b587d82c874df275f147eb461f6389163547e4e Mon Sep 17 00:00:00 2001 From: frosty Date: Mon, 3 Aug 2026 23:21:41 -0400 Subject: feat: add yacy support --- example-config.ini | 5 +- src/Config.c | 4 ++ src/Config.h | 2 + src/Main.c | 18 +++++++ src/Scraping/Scraping.c | 36 +++++++++---- src/Scraping/Scraping.h | 2 + src/Scraping/ScrapingHttp.c | 13 +++-- src/Scraping/ScrapingParsers.c | 116 +++++++++++++++++++++++++++++++++++++---- 8 files changed, 170 insertions(+), 26 deletions(-) diff --git a/example-config.ini b/example-config.ini index 2760c53..55413a8 100644 --- a/example-config.ini +++ b/example-config.ini @@ -31,8 +31,11 @@ port = 8087 [engines] # Use * for all engines, or specify comma-separated list (e.g., ddg,yahoo) # Use *,-engine to exclude specific engines (e.g., *,-startpage) -# Available engines: ddg, startpage, yahoo, mojeek +# Available engines: ddg, startpage, yahoo, mojeek, yacy engines="*" +# YaCy instance to use for the yacy engine (optional, default: http://127.0.0.1:8090) +# Requires "yacy" to be in the engines list above. +#yacy_instance = http://127.0.0.1:8090 [rate_limit] # Rate limit searches per interval diff --git a/src/Config.c b/src/Config.c index 9883d45..bde76fe 100644 --- a/src/Config.c +++ b/src/Config.c @@ -100,6 +100,10 @@ int load_config(const char *filename, Config *config) { if (strcmp(key, "engines") == 0) { strncpy(config->engines, value, sizeof(config->engines) - 1); config->engines[sizeof(config->engines) - 1] = '\0'; + } else if (strcmp(key, "yacy_instance") == 0) { + strncpy(config->yacy_instance, value, + sizeof(config->yacy_instance) - 1); + config->yacy_instance[sizeof(config->yacy_instance) - 1] = '\0'; } } else if (strcmp(section, "rate_limit") == 0) { if (strcmp(key, "search_requests") == 0) { diff --git a/src/Config.h b/src/Config.h index 25bd978..f0bdf4b 100644 --- a/src/Config.h +++ b/src/Config.h @@ -8,6 +8,7 @@ #define DEFAULT_CACHE_TTL_INFOBOX 86400 #define DEFAULT_CACHE_TTL_IMAGE 604800 #define DEFAULT_MAX_PROXY_RETRIES 3 +#define DEFAULT_YACY_INSTANCE "http://127.0.0.1:8090" #define BUFFER_SIZE_SMALL 256 #define BUFFER_SIZE_MEDIUM 512 @@ -46,6 +47,7 @@ typedef struct { int cache_ttl_infobox; int cache_ttl_image; char engines[512]; + char yacy_instance[512]; int rate_limit_search_requests; int rate_limit_search_interval; int rate_limit_images_requests; diff --git a/src/Main.c b/src/Main.c index d7ff185..868244c 100644 --- a/src/Main.c +++ b/src/Main.c @@ -78,6 +78,7 @@ int main() { .cache_ttl_infobox = DEFAULT_CACHE_TTL_INFOBOX, .cache_ttl_image = DEFAULT_CACHE_TTL_IMAGE, .engines = "", + .yacy_instance = "", .rate_limit_search_requests = 0, .rate_limit_search_interval = 0, .rate_limit_images_requests = 0, @@ -101,6 +102,23 @@ int main() { } apply_engines_config(cfg.engines); + configure_yacy_engine(cfg.yacy_instance); + + if (cfg.yacy_instance[0] != '\0') { + int yacy_enabled = 0; + for (int i = 0; i < ENGINE_COUNT; i++) { + if (strcmp(ENGINE_REGISTRY[i].id, "yacy") == 0) { + yacy_enabled = ENGINE_REGISTRY[i].enabled; + break; + } + } + if (!yacy_enabled) { + fprintf( + stderr, + "[INFO] YaCy instance configured but the yacy engine is not " + "enabled (add it to the engines list, e.g. engines=\"*,yacy\")\n"); + } + } if (cache_init(cfg.cache_dir) != 0) { fprintf(stderr, diff --git a/src/Scraping/Scraping.c b/src/Scraping/Scraping.c index b81c216..5c51ca5 100644 --- a/src/Scraping/Scraping.c +++ b/src/Scraping/Scraping.c @@ -4,6 +4,7 @@ #include "Config.h" #include #include +#include #include #include #include @@ -24,7 +25,8 @@ static int response_is_startpage_captcha(const ScrapeJob *job, return response_contains(response, "Startpage Captcha") || response_contains(response, "Startpage Captcha") || - response_contains(response, "/static-pages-assets/page-data/captcha/") || + response_contains(response, + "/static-pages-assets/page-data/captcha/") || response_contains(response, ">Startpage Blocked"); } @@ -39,7 +41,8 @@ static int response_looks_like_results_page(const ScrapeJob *job, } if (strcmp(job->engine->name, "Startpage") == 0) { - return response_contains(response, "Startpage Search Results") || + return response_contains(response, + "Startpage Search Results") || response_contains(response, "class=\"w-gl") || response_contains(response, "data-testid=\"gl-title-link\""); } @@ -55,6 +58,11 @@ static int response_looks_like_results_page(const ScrapeJob *job, response_contains(response, "Mojeek Search"); } + if (strcmp(job->engine->name, "YaCy") == 0) { + return response_contains(response, "engine->is_xml) { + doc = xmlReadMemory(response, response_size, NULL, NULL, + XML_PARSE_RECOVER | XML_PARSE_NOERROR | + XML_PARSE_NOWARNING); + } else { + doc = htmlReadMemory(response, response_size, NULL, NULL, + HTML_PARSE_RECOVER | HTML_PARSE_NOERROR | + HTML_PARSE_NOWARNING); + } if (!doc) { job->status = SCRAPE_STATUS_FETCH_ERROR; return; } - job->results_count = - job->engine->parser(job->engine->name, doc, job->out_results, - job->max_results); + job->results_count = job->engine->parser(job->engine->name, doc, + job->out_results, job->max_results); xmlFreeDoc(doc); if (job->results_count > 0) { @@ -200,7 +214,8 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) { return -1; } - for (char *p = encoded_query + strlen(encoded_query) - 3; p >= encoded_query; p--) { + for (char *p = encoded_query + strlen(encoded_query) - 3; p >= encoded_query; + p--) { if (p[0] == '%' && p[1] == '2' && p[2] == '0') { *p = '+'; memmove(p + 1, p + 3, strlen(p + 3) + 1); @@ -279,8 +294,7 @@ int should_retry(ScrapeJob *jobs, int num_jobs) { int scrape_engines_parallel(ScrapeJob *jobs, int num_jobs) { int retries = 0; -retry: - ; +retry:; CURLM *multi_handle = curl_multi_init(); if (!multi_handle) return -1; diff --git a/src/Scraping/Scraping.h b/src/Scraping/Scraping.h index be65e5a..14e6e55 100644 --- a/src/Scraping/Scraping.h +++ b/src/Scraping/Scraping.h @@ -26,6 +26,7 @@ typedef struct { int page_base; ParserFunc parser; int enabled; + int is_xml; } SearchEngine; typedef struct { @@ -59,6 +60,7 @@ typedef struct { extern SearchEngine ENGINE_REGISTRY[]; extern const int ENGINE_COUNT; void apply_engines_config(const char *engines_str); +void configure_yacy_engine(const char *instance); size_t write_memory_callback(void *contents, size_t size, size_t nmemb, void *userp); diff --git a/src/Scraping/ScrapingHttp.c b/src/Scraping/ScrapingHttp.c index 1a6a292..89f4853 100644 --- a/src/Scraping/ScrapingHttp.c +++ b/src/Scraping/ScrapingHttp.c @@ -89,11 +89,14 @@ struct curl_slist *build_request_headers(const char *host_header, struct curl_slist *headers = NULL; char host_buf[BUFFER_SIZE_MEDIUM], ref_buf[BUFFER_SIZE_MEDIUM]; - snprintf(host_buf, sizeof(host_buf), "Host: %s", host_header); - snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer); - - headers = curl_slist_append(headers, host_buf); - headers = curl_slist_append(headers, ref_buf); + if (host_header && host_header[0] != '\0') { + snprintf(host_buf, sizeof(host_buf), "Host: %s", host_header); + headers = curl_slist_append(headers, host_buf); + } + if (referer && referer[0] != '\0') { + snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer); + headers = curl_slist_append(headers, ref_buf); + } headers = curl_slist_append( headers, "Accept: " diff --git a/src/Scraping/ScrapingParsers.c b/src/Scraping/ScrapingParsers.c index 96aaded..2e5689b 100644 --- a/src/Scraping/ScrapingParsers.c +++ b/src/Scraping/ScrapingParsers.c @@ -249,8 +249,8 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc, if (!ctx) return 0; - xmlXPathObjectPtr obj = - xml_xpath_eval(ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]"); + xmlXPathObjectPtr obj = xml_xpath_eval( + ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]"); if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) { free_xpath_objects(ctx, obj); @@ -268,18 +268,17 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc, xmlNodePtr result_node = obj->nodesetval->nodeTab[i]; ctx->node = result_node; - xmlXPathObjectPtr link_obj = - xml_xpath_eval(ctx, ".//a[@class='title']"); + xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, ".//a[@class='title']"); char *url = (link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0) ? (char *)xmlGetProp(link_obj->nodesetval->nodeTab[0], (xmlChar *)"href") : NULL; - char *title = (link_obj && link_obj->nodesetval && - link_obj->nodesetval->nodeNr > 0) - ? xml_node_content(link_obj->nodesetval->nodeTab[0]) - : NULL; + char *title = + (link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0) + ? xml_node_content(link_obj->nodesetval->nodeTab[0]) + : NULL; xmlXPathObjectPtr snippet_obj = xml_xpath_eval(ctx, ".//p[@class='s']"); char *snippet_text = @@ -310,6 +309,90 @@ static int parse_yahoo(const char *engine_name, xmlDocPtr doc, static int parse_mojeek(const char *engine_name, xmlDocPtr doc, SearchResult **out_results, int max_results); +static int parse_yacy(const char *engine_name, xmlDocPtr doc, + SearchResult **out_results, int max_results) { + (void)engine_name; + int found_count = 0; + + xmlXPathContextPtr ctx = create_xpath_context(doc); + if (!ctx) + return 0; + + xmlXPathObjectPtr obj = xml_xpath_eval(ctx, "//item"); + + if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) { + free_xpath_objects(ctx, obj); + return 0; + } + + int num_items = obj->nodesetval->nodeNr; + *out_results = alloc_results_array(num_items, max_results); + if (!*out_results) { + free_xpath_objects(ctx, obj); + return 0; + } + + for (int i = 0; i < num_items && found_count < max_results; i++) { + xmlNodePtr item_node = obj->nodesetval->nodeTab[i]; + ctx->node = item_node; + + xmlXPathObjectPtr title_obj = xml_xpath_eval(ctx, "./title"); + char *title = (title_obj && title_obj->nodesetval && + title_obj->nodesetval->nodeNr > 0) + ? xml_node_content(title_obj->nodesetval->nodeTab[0]) + : NULL; + + xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, "./link"); + char *url = + (link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0) + ? xml_node_content(link_obj->nodesetval->nodeTab[0]) + : NULL; + + xmlXPathObjectPtr desc_obj = xml_xpath_eval(ctx, "./description"); + char *snippet_text = + (desc_obj && desc_obj->nodesetval && desc_obj->nodesetval->nodeNr > 0) + ? xml_node_content(desc_obj->nodesetval->nodeTab[0]) + : NULL; + + if (url && title) { + assign_result(&(*out_results)[found_count], url, title, snippet_text, 0); + found_count++; + } + + free_xml_node_list(title, url, snippet_text); + if (title_obj) + xmlXPathFreeObject(title_obj); + if (link_obj) + xmlXPathFreeObject(link_obj); + if (desc_obj) + xmlXPathFreeObject(desc_obj); + } + + ctx->node = NULL; + free_xpath_objects(ctx, obj); + return found_count; +} + +static char yacy_base_url[BUFFER_SIZE_LARGE]; + +void configure_yacy_engine(const char *instance) { + if (!instance || instance[0] == '\0') + instance = DEFAULT_YACY_INSTANCE; + + snprintf(yacy_base_url, sizeof(yacy_base_url), + "%s/" + "yacysearch.rss?resource=global&urlmaskfilter=.*&prefermaskfilter=&" + "nav=all&maximumRecords=%d&query=", + instance, MAX_RESULTS_PER_ENGINE); + + for (int i = 0; i < ENGINE_COUNT; i++) { + if (strcmp(ENGINE_REGISTRY[i].id, "yacy") == 0) { + ENGINE_REGISTRY[i].base_url = yacy_base_url; + break; + } + } +} + SearchEngine ENGINE_REGISTRY[] = { {.id = "ddg", .name = "DuckDuckGo Lite", @@ -350,7 +433,18 @@ SearchEngine ENGINE_REGISTRY[] = { .page_multiplier = 10, .page_base = 1, .parser = parse_mojeek, - .enabled = 1}}; + .enabled = 1}, + {.id = "yacy", + .name = "YaCy", + .base_url = "", + .host_header = NULL, + .referer = NULL, + .page_param = "startRecord", + .page_multiplier = 10, + .page_base = 0, + .parser = parse_yacy, + .enabled = 0, + .is_xml = 1}}; const int ENGINE_COUNT = sizeof(ENGINE_REGISTRY) / sizeof(SearchEngine); @@ -373,6 +467,8 @@ static int engine_id_compare(const char *engine_id, const char *config_id) { void apply_engines_config(const char *engines_str) { if (!engines_str || engines_str[0] == '\0') { for (int i = 0; i < ENGINE_COUNT; i++) { + if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy")) + continue; ENGINE_REGISTRY[i].enabled = 1; } return; @@ -395,6 +491,8 @@ void apply_engines_config(const char *engines_str) { if (strcmp(token, "*") == 0) { for (int i = 0; i < ENGINE_COUNT; i++) { + if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy")) + continue; ENGINE_REGISTRY[i].enabled = 1; } } else if (token[0] == '-' && token[1] != '\0') { -- cgit v1.3 From 11acc93f4357b86e2ca3a74e549193d84ab685f4 Mon Sep 17 00:00:00 2001 From: frosty Date: Tue, 4 Aug 2026 00:51:28 -0400 Subject: fix: remove nix support --- README.md | 26 ----------- flake.lock | 46 -------------------- flake.nix | 76 --------------------------------- module.nix | 142 ------------------------------------------------------------- 4 files changed, 290 deletions(-) delete mode 100644 flake.lock delete mode 100644 flake.nix delete mode 100644 module.nix diff --git a/README.md b/README.md index 00f2173..7798229 100644 --- a/README.md +++ b/README.md @@ -47,32 +47,6 @@ On Alpine, `shadow` is needed for the user creation process during the install. # xbps-install -S libxml2-devel libcurl-devel ``` -### NixOS -Add the flake to your inputs and import the module. That is all you need. -Here's an example of using the modules in a flake: -```nix -# flake.nix -{ - inputs = { - omnisearch = { - url = "git+https://git.bwaaa.monster/omnisearch"; - inputs.nixpkgs.follows = "nixpkgs"; - }; - }; - - outputs = { self, nixpkgs, omnisearch, ... }: { - nixosConfigurations.mySystem = nixpkgs.lib.nixosSystem { - modules = [ - omnisearch.nixosModules.default - { - services.omnisearch.enable = true; - } - ]; - }; - }; -} -``` - ### macOS (Homebrew) ``` $ brew install libxml2 curl openssl pkg-config diff --git a/flake.lock b/flake.lock deleted file mode 100644 index 330fc96..0000000 --- a/flake.lock +++ /dev/null @@ -1,46 +0,0 @@ -{ - "nodes": { - "beaker-src": { - "flake": false, - "locked": { - "lastModified": 1775244490, - "narHash": "sha256-4TJv7X6D0l4rEbTRKf47gU43L8G5uJgxxtsqMkVixQY=", - "ref": "refs/heads/master", - "rev": "3fab89ecf8f4c664477a82add660d28db87357b4", - "revCount": 27, - "shallow": false, - "type": "git", - "url": "https://git.bwaaa.monster/beaker" - }, - "original": { - "shallow": false, - "type": "git", - "url": "https://git.bwaaa.monster/beaker" - } - }, - "nixpkgs": { - "locked": { - "lastModified": 1773734432, - "narHash": "sha256-IF5ppUWh6gHGHYDbtVUyhwy/i7D261P7fWD1bPefOsw=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "cda48547b432e8d3b18b4180ba07473762ec8558", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "root": { - "inputs": { - "beaker-src": "beaker-src", - "nixpkgs": "nixpkgs" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/flake.nix b/flake.nix deleted file mode 100644 index 1a7140e..0000000 --- a/flake.nix +++ /dev/null @@ -1,76 +0,0 @@ -{ - inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; - beaker-src = { - url = "git+https://git.bwaaa.monster/beaker?shallow=0"; - flake = false; - }; - }; - - outputs = - { - self, - nixpkgs, - beaker-src, - }: - let - supportedSystems = [ - "x86_64-linux" - "aarch64-linux" - ]; - forAllSystems = nixpkgs.lib.genAttrs supportedSystems; - in - { - formatter = forAllSystems (system: nixpkgs.legacyPackages.${system}.nixfmt); - packages = forAllSystems ( - system: - let - pkgs = import nixpkgs { inherit system; }; - - beaker = pkgs.stdenv.mkDerivation { - pname = "beaker"; - version = "git"; - src = beaker-src; - makeFlags = [ - "INSTALL_PREFIX=$(out)/" - "LDCONFIG=true" - ]; - }; - in - { - default = pkgs.stdenv.mkDerivation { - pname = "omnisearch"; - version = "git"; - src = ./.; - - buildInputs = [ - pkgs.libxml2.dev - pkgs.curl.dev - pkgs.openssl - beaker - ]; - - preBuild = '' - makeFlagsArray+=( - "PREFIX=$out" - "CFLAGS=-Wall -Wextra -O2 -Isrc -I${pkgs.libxml2.dev}/include/libxml2" - "LIBS=-lbeaker -lcurl -lxml2 -lpthread -lm -lssl -lcrypto" - ) - ''; - - installPhase = '' - mkdir -p $out/bin $out/share/omnisearch - install -Dm755 bin/omnisearch $out/bin/omnisearch - cp -r templates static locales -t $out/share/omnisearch/ - ''; - - meta = { - description = "Lightweight metasearch engine in C"; - platforms = pkgs.lib.platforms.linux; - }; - }; - } - ); - nixosModules.default = import ./module.nix self; - }; -} diff --git a/module.nix b/module.nix deleted file mode 100644 index c0c7825..0000000 --- a/module.nix +++ /dev/null @@ -1,142 +0,0 @@ -self: - -{ - config, - lib, - pkgs, - ... -}: - -let - cfg = config.services.omnisearch; - pkg = cfg.package; - - finalConfigFile = - if cfg.configFile != null then - cfg.configFile - else - pkgs.writeText "omnisearch.ini" '' - [server] - host = ${cfg.settings.server.host} - port = ${toString cfg.settings.server.port} - domain = ${cfg.settings.server.domain} - ${lib.optionalString (cfg.settings.server.locale != null) "locale = ${cfg.settings.server.locale}"} - - [proxy] - ${lib.optionalString (cfg.settings.proxy.proxy != null) "proxy = \"${cfg.settings.proxy.proxy}\""} - ${lib.optionalString ( - cfg.settings.proxy.list_file != null - ) "list_file = ${cfg.settings.proxy.list_file}"} - max_retries = ${toString cfg.settings.proxy.max_retries} - randomize_username = ${lib.boolToString cfg.settings.proxy.randomize_username} - randomize_password = ${lib.boolToString cfg.settings.proxy.randomize_password} - - [cache] - dir = ${cfg.settings.cache.dir} - ttl_search = ${toString cfg.settings.cache.ttl_search} - ttl_infobox = ${toString cfg.settings.cache.ttl_infobox} - ''; -in -{ - options.services.omnisearch = { - enable = lib.mkEnableOption "OmniSearch metasearch engine"; - - package = lib.mkOption { - type = lib.types.package; - default = self.packages.${pkgs.stdenv.hostPlatform.system}.default; - description = "The omnisearch package to use."; - }; - - configFile = lib.mkOption { - type = lib.types.nullOr lib.types.path; - default = null; - description = "Path to a custom config.ini. Overrides 'settings'."; - }; - - settings = { - server = { - host = lib.mkOption { - type = lib.types.str; - default = "0.0.0.0"; - }; - port = lib.mkOption { - type = lib.types.port; - default = 8087; - }; - domain = lib.mkOption { - type = lib.types.str; - default = "http://localhost:${toString cfg.settings.server.port}"; - }; - locale = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - }; - }; - proxy = { - proxy = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - }; - list_file = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - }; - max_retries = lib.mkOption { - type = lib.types.int; - default = 3; - }; - randomize_username = lib.mkOption { - type = lib.types.bool; - default = true; - }; - randomize_password = lib.mkOption { - type = lib.types.bool; - default = true; - }; - }; - cache = { - dir = lib.mkOption { - type = lib.types.str; - default = "/var/cache/omnisearch"; - }; - ttl_search = lib.mkOption { - type = lib.types.int; - default = 3600; - }; - ttl_infobox = lib.mkOption { - type = lib.types.int; - default = 86400; - }; - }; - }; - }; - - config = lib.mkIf cfg.enable { - systemd.services.omnisearch = { - description = "OmniSearch Service"; - after = [ "network.target" ]; - wantedBy = [ "multi-user.target" ]; - - serviceConfig = { - ExecStart = "${pkg}/bin/omnisearch"; - - WorkingDirectory = "/var/lib/omnisearch"; - StateDirectory = "omnisearch"; - CacheDirectory = "omnisearch"; - - BindReadOnlyPaths = [ - "${pkg}/share/omnisearch/templates:/var/lib/omnisearch/templates" - "${pkg}/share/omnisearch/static:/var/lib/omnisearch/static" - "${pkg}/share/omnisearch/locales:/var/lib/omnisearch/locales" - "${finalConfigFile}:/var/lib/omnisearch/config.ini" - ]; - - DynamicUser = true; - ProtectSystem = "strict"; - ProtectHome = true; - PrivateTmp = true; - Restart = "always"; - }; - }; - }; -} -- cgit v1.3