diff options
Diffstat (limited to 'README.ORG')
| -rw-r--r-- | README.ORG | 31 |
1 files changed, 31 insertions, 0 deletions
diff --git a/README.ORG b/README.ORG new file mode 100644 index 0000000..ea362f3 --- /dev/null +++ b/README.ORG @@ -0,0 +1,31 @@ +* crawler +** sqlite +urls: +|---------+----------+--------| +| id: num | url: str | status | +|---------+----------+--------| +status: one of pending/done/failed + +links: +|--------------+------------| +| from: url_id | to: url_id | +|--------------+------------| + + +queue +| url1 | +| url2 | +| url3 | + +thread: ++ lock queue ++ dequeue url_a ++ unlock queue ++ set url_a status to pending ++ fetch page ++ filter out urls ++ for each url_b in urls: + + insert url_b into sqlite or ignore (if already exists) + + insert link (url_a -> url_b) into sqlite + + enqueue url_b if it did not exist in sqlite (lock -> enqueue -> unlock) ++ set url_a status to done |