blob: e33f529b465954b4fd9b093d5cd310e66617e59d (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
|
#ifndef SCRAPING_H
#define SCRAPING_H
#include <libxml/HTMLparser.h>
#include <curl/curl.h>
typedef struct {
char *url;
char *title;
char *snippet;
} SearchResult;
typedef int (*ParserFunc)(const char *engine_name, xmlDocPtr doc,
SearchResult **out_results, int max_results);
typedef struct {
const char *name;
const char *base_url;
const char *host_header;
const char *referer;
const char *page_param;
int page_multiplier;
int page_base;
ParserFunc parser;
} SearchEngine;
typedef struct {
char *memory;
size_t size;
size_t capacity;
} MemoryBuffer;
typedef struct {
const SearchEngine *engine;
char *query;
SearchResult **out_results;
int max_results;
int page;
CURL *handle;
MemoryBuffer response;
int results_count;
} ScrapeJob;
extern const SearchEngine ENGINE_REGISTRY[];
extern const int ENGINE_COUNT;
int scrape_engine(const SearchEngine *engine, const char *query,
SearchResult **out_results, int max_results);
int scrape_engines_parallel(ScrapeJob *jobs, int num_jobs);
#endif
|
