From 47f16db1909d185f7a6c5987226f64f0e2788262 Mon Sep 17 00:00:00 2001 From: frosty Date: Thu, 22 Jan 2026 12:57:27 -0500 Subject: scraping now more efficient blehhh --- src/Scraping/Scraping.h | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) (limited to 'src/Scraping/Scraping.h') diff --git a/src/Scraping/Scraping.h b/src/Scraping/Scraping.h index 7ad4d59..43e22d9 100644 --- a/src/Scraping/Scraping.h +++ b/src/Scraping/Scraping.h @@ -2,6 +2,7 @@ #define SCRAPING_H #include +#include #define LOG_INFO(msg, ...) fprintf(stderr, "[INFO] " msg "\n", ##__VA_ARGS__) #define LOG_WARN(msg, ...) fprintf(stderr, "[WARN] " msg "\n", ##__VA_ARGS__) @@ -25,10 +26,28 @@ typedef struct { ParserFunc parser; } SearchEngine; +typedef struct { + char *memory; + size_t size; + size_t capacity; +} MemoryBuffer; + +typedef struct { + const SearchEngine *engine; + char *query; + SearchResult **out_results; + int max_results; + CURL *handle; + MemoryBuffer response; + int results_count; +} ScrapeJob; + extern const SearchEngine ENGINE_REGISTRY[]; extern const int ENGINE_COUNT; int scrape_engine(const SearchEngine *engine, const char *query, SearchResult **out_results, int max_results); -#endif +int scrape_engines_parallel(ScrapeJob *jobs, int num_jobs); + +#endif \ No newline at end of file -- cgit v1.2.3