scraping now more efficient blehhh

1 files changed, 20 insertions, 1 deletions

diff --git a/src/Scraping/Scraping.h b/src/Scraping/Scraping.h
index 7ad4d59..43e22d9 100644
--- a/src/Scraping/Scraping.h
+++ b/src/Scraping/Scraping.h
@@ -2,6 +2,7 @@
 #define SCRAPING_H
 
 #include <libxml/HTMLparser.h>
+#include <curl/curl.h>
 
 #define LOG_INFO(msg, ...) fprintf(stderr, "[INFO] " msg "\n", ##__VA_ARGS__)
 #define LOG_WARN(msg, ...) fprintf(stderr, "[WARN] " msg "\n", ##__VA_ARGS__)
@@ -25,10 +26,28 @@ typedef struct {
   ParserFunc parser;
 } SearchEngine;
 
+typedef struct {
+  char *memory;
+  size_t size;
+  size_t capacity;
+} MemoryBuffer;
+
+typedef struct {
+  const SearchEngine *engine;
+  char *query;
+  SearchResult **out_results;
+  int max_results;
+  CURL *handle;
+  MemoryBuffer response;
+  int results_count;
+} ScrapeJob;
+
 extern const SearchEngine ENGINE_REGISTRY[];
 extern const int ENGINE_COUNT;
 
 int scrape_engine(const SearchEngine *engine, const char *query,
                   SearchResult **out_results, int max_results);
 
-#endif
+int scrape_engines_parallel(ScrapeJob *jobs, int num_jobs);
+
+#endif
\ No newline at end of file