1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
|
#ifndef SCRAPING_H
#define SCRAPING_H
#include <libxml/HTMLparser.h>
#include <curl/curl.h>
#define LOG_INFO(msg, ...) fprintf(stderr, "[INFO] " msg "\n", ##__VA_ARGS__)
#define LOG_WARN(msg, ...) fprintf(stderr, "[WARN] " msg "\n", ##__VA_ARGS__)
#define LOG_DEBUG(msg, ...) fprintf(stderr, "[DEBUG] " msg "\n", ##__VA_ARGS__)
#define LOG_ERROR(msg, ...) fprintf(stderr, "[ERROR] " msg "\n", ##__VA_ARGS__)
typedef struct {
char *url;
char *title;
char *snippet;
} SearchResult;
typedef int (*ParserFunc)(const char *engine_name, xmlDocPtr doc,
SearchResult **out_results, int max_results);
typedef struct {
const char *name;
const char *base_url;
const char *host_header;
const char *referer;
ParserFunc parser;
} SearchEngine;
typedef struct {
char *memory;
size_t size;
size_t capacity;
} MemoryBuffer;
typedef struct {
const SearchEngine *engine;
char *query;
SearchResult **out_results;
int max_results;
CURL *handle;
MemoryBuffer response;
int results_count;
} ScrapeJob;
extern const SearchEngine ENGINE_REGISTRY[];
extern const int ENGINE_COUNT;
int scrape_engine(const SearchEngine *engine, const char *query,
SearchResult **out_results, int max_results);
int scrape_engines_parallel(ScrapeJob *jobs, int num_jobs);
#endif
|
