Pathogen
/
monolith
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Give option for only crawling some boards

This commit is contained in:
Mark Veidemanis 2022-12-22 07:20:26 +00:00
parent ebaf8c765d
commit 0ab67becff
Signed by: m
GPG Key ID: 5ACFCEED46C0904F
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
sources/ch4.py
View File

@ -25,6 +25,9 @@ CRAWL_DELAY = int(getenv("MONOLITH_CH4_CRAWL_DELAY", 5))
# Semaphore value ? # Semaphore value ?
THREADS_SEMAPHORE = int(getenv("MONOLITH_CH4_THREADS_SEMAPHORE", 1000)) THREADS_SEMAPHORE = int(getenv("MONOLITH_CH4_THREADS_SEMAPHORE", 1000))
# Boards to crawl
BOARDS = getenv("MONOLITH_CH4_BOARDS", "").split(",")
# CONFIGURATION END # # CONFIGURATION END #
@ -58,7 +61,10 @@ class Chan4(object):
self.log.debug(f"Decoded hash key: {self.hash_key}") self.log.debug(f"Decoded hash key: {self.hash_key}")
async def run(self): async def run(self):
if not BOARDS:
await self.get_board_list() await self.get_board_list()
else:
self.boards = BOARDS
while True: while True:
await self.get_thread_lists(self.boards) await self.get_thread_lists(self.boards)
await asyncio.sleep(CRAWL_DELAY) await asyncio.sleep(CRAWL_DELAY)