diff options
| author | Felix Perktold <Felix.Perktold@student.uibk.ac.at> | 2026-07-22 03:17:51 +0200 |
|---|---|---|
| committer | Felix Perktold <Felix.Perktold@student.uibk.ac.at> | 2026-07-22 03:17:51 +0200 |
| commit | 0d0f36ae5292c918042b843961d402056076b9a8 (patch) | |
| tree | 00288367287108a121a84f3f50d4808e44f2da8b /README.ORG | |
init
Diffstat (limited to 'README.ORG')
| -rw-r--r-- | README.ORG | 31 |
1 files changed, 31 insertions, 0 deletions
diff --git a/README.ORG b/README.ORG new file mode 100644 index 0000000..ea362f3 --- /dev/null +++ b/README.ORG @@ -0,0 +1,31 @@ +* crawler +** sqlite +urls: +|---------+----------+--------| +| id: num | url: str | status | +|---------+----------+--------| +status: one of pending/done/failed + +links: +|--------------+------------| +| from: url_id | to: url_id | +|--------------+------------| + + +queue +| url1 | +| url2 | +| url3 | + +thread: ++ lock queue ++ dequeue url_a ++ unlock queue ++ set url_a status to pending ++ fetch page ++ filter out urls ++ for each url_b in urls: + + insert url_b into sqlite or ignore (if already exists) + + insert link (url_a -> url_b) into sqlite + + enqueue url_b if it did not exist in sqlite (lock -> enqueue -> unlock) ++ set url_a status to done |