filter urls betterHEADmaster

1 files changed, 7 insertions, 2 deletions

diff --git a/main.c b/main.c
index d3a92fe..6b2e48f 100644
--- a/main.c
+++ b/main.c
@@ -199,6 +199,7 @@ static size_t curl_callback(char *str, size_t size, size_t nmemb, void *ctx) {
 	memcpy(buf, str, n);
 	buf[n] = '\0';
 	char *cursor = buf;
+
 	regmatch_t match;
 
 	// look for urls
@@ -208,7 +209,11 @@ static size_t curl_callback(char *str, size_t size, size_t nmemb, void *ctx) {
                 memcpy(url, cursor + match.rm_so, len);
                 url[len] = '\0';
 
-		//printf("found url: %s\n", url);
+		if (len > 300) {
+			free(url);
+			cursor += match.rm_eo;
+			continue;
+		}
 
 		//insert url into db
 		int inserted = db_insert_url(db, url);
@@ -277,7 +282,7 @@ int main(void) {
 	fill_queue(DB);
 
 	//init regex
-	const char *pattern = "https?://[^[:space:]\"<>#]+";
+	const char *pattern = "https?://[^[:space:]\"'<>#?]+";
 	if (regcomp(&regex, pattern, REG_EXTENDED) != 0) {
 		printf("Failed to compile regex\n");
 		return 1;