From ab9bb3793d7885387f6552912fedf85b1eba95a6 Mon Sep 17 00:00:00 2001 From: Felix Perktold Date: Wed, 22 Jul 2026 14:45:45 +0200 Subject: filter urls better --- main.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/main.c b/main.c index d3a92fe..6b2e48f 100644 --- a/main.c +++ b/main.c @@ -199,6 +199,7 @@ static size_t curl_callback(char *str, size_t size, size_t nmemb, void *ctx) { memcpy(buf, str, n); buf[n] = '\0'; char *cursor = buf; + regmatch_t match; // look for urls @@ -208,7 +209,11 @@ static size_t curl_callback(char *str, size_t size, size_t nmemb, void *ctx) { memcpy(url, cursor + match.rm_so, len); url[len] = '\0'; - //printf("found url: %s\n", url); + if (len > 300) { + free(url); + cursor += match.rm_eo; + continue; + } //insert url into db int inserted = db_insert_url(db, url); @@ -277,7 +282,7 @@ int main(void) { fill_queue(DB); //init regex - const char *pattern = "https?://[^[:space:]\"<>#]+"; + const char *pattern = "https?://[^[:space:]\"'<>#?]+"; if (regcomp(®ex, pattern, REG_EXTENDED) != 0) { printf("Failed to compile regex\n"); return 1; -- cgit v1.2.3