From 7b46d74f9d9e373ab71ffba477e4085c6c58cb59 Mon Sep 17 00:00:00 2001 From: cathuyenphan Date: Mon, 5 May 2025 23:48:05 -0700 Subject: [PATCH 1/2] deleted venv and install dependencies --- SpringCrawler/crawler.bat | 4 ---- SpringCrawler/crawler.sh | 4 ---- 2 files changed, 8 deletions(-) diff --git a/SpringCrawler/crawler.bat b/SpringCrawler/crawler.bat index f369eb6..67b3cb6 100644 --- a/SpringCrawler/crawler.bat +++ b/SpringCrawler/crawler.bat @@ -19,10 +19,6 @@ if not exist venv ( python -m venv venv ) -call venv\Scripts\activate.bat -pip install --upgrade pip -pip install -r requirements.txt - set SCRAPY_CMND=scrapy crawl url -a seed_file=%SEED_FILE% -a num_pages=%NUM_PAGES% if not "%DEPTH%"=="" ( diff --git a/SpringCrawler/crawler.sh b/SpringCrawler/crawler.sh index ecfb224..eb21c9f 100644 --- a/SpringCrawler/crawler.sh +++ b/SpringCrawler/crawler.sh @@ -15,10 +15,6 @@ if [ ! -d "venv" ]; then python3 -m venv venv fi -source venv/bin/activate -pip install --upgrade pip -pip install -r requirements.txt - SCRAPY_CMND="scrapy crawl url -a seed_file=$SEED_FILE -a num_pages=$NUM_PAGES" if [ -n "$DEPTH" ]; then From 099272106f52d5eef026c9bef2a5771b03caa025 Mon Sep 17 00:00:00 2001 From: cathuyenphan Date: Mon, 5 May 2025 23:51:58 -0700 Subject: [PATCH 2/2] deleted stuff --- SpringCrawler/crawler.bat | 4 ---- SpringCrawler/crawler.sh | 4 ---- 2 files changed, 8 deletions(-) diff --git a/SpringCrawler/crawler.bat b/SpringCrawler/crawler.bat index 67b3cb6..501e188 100644 --- a/SpringCrawler/crawler.bat +++ b/SpringCrawler/crawler.bat @@ -15,10 +15,6 @@ set NUM_PAGES=%~2 set DEPTH=%~3 set OUTPUT_DIR=%~4 -if not exist venv ( - python -m venv venv -) - set SCRAPY_CMND=scrapy crawl url -a seed_file=%SEED_FILE% -a num_pages=%NUM_PAGES% if not "%DEPTH%"=="" ( diff --git a/SpringCrawler/crawler.sh b/SpringCrawler/crawler.sh index eb21c9f..1b86592 100644 --- a/SpringCrawler/crawler.sh +++ b/SpringCrawler/crawler.sh @@ -11,10 +11,6 @@ NUM_PAGES=$2 DEPTH=$3 OUTPUT_DIR=$4 -if [ ! -d "venv" ]; then - python3 -m venv venv -fi - SCRAPY_CMND="scrapy crawl url -a seed_file=$SEED_FILE -a num_pages=$NUM_PAGES" if [ -n "$DEPTH" ]; then