aboutsummaryrefslogtreecommitdiff
path: root/src/Scraping/Scraping.h
diff options
context:
space:
mode:
Diffstat (limited to 'src/Scraping/Scraping.h')
-rw-r--r--src/Scraping/Scraping.h21
1 files changed, 20 insertions, 1 deletions
diff --git a/src/Scraping/Scraping.h b/src/Scraping/Scraping.h
index 7ad4d59..43e22d9 100644
--- a/src/Scraping/Scraping.h
+++ b/src/Scraping/Scraping.h
@@ -2,6 +2,7 @@
#define SCRAPING_H
#include <libxml/HTMLparser.h>
+#include <curl/curl.h>
#define LOG_INFO(msg, ...) fprintf(stderr, "[INFO] " msg "\n", ##__VA_ARGS__)
#define LOG_WARN(msg, ...) fprintf(stderr, "[WARN] " msg "\n", ##__VA_ARGS__)
@@ -25,10 +26,28 @@ typedef struct {
ParserFunc parser;
} SearchEngine;
+typedef struct {
+ char *memory;
+ size_t size;
+ size_t capacity;
+} MemoryBuffer;
+
+typedef struct {
+ const SearchEngine *engine;
+ char *query;
+ SearchResult **out_results;
+ int max_results;
+ CURL *handle;
+ MemoryBuffer response;
+ int results_count;
+} ScrapeJob;
+
extern const SearchEngine ENGINE_REGISTRY[];
extern const int ENGINE_COUNT;
int scrape_engine(const SearchEngine *engine, const char *query,
SearchResult **out_results, int max_results);
-#endif
+int scrape_engines_parallel(ScrapeJob *jobs, int num_jobs);
+
+#endif \ No newline at end of file