HKJC repo is a project fro analyzing the horse racing data from HKJC
- Download Miniconda3
wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
chmod +x Miniconda3-latest-Linux-x86_64.sh
./Miniconda3-latest-Linux-x86_64.sh
export PATH=~/miniconda/bin:$PATH
- Download PostgreSQL 11
sudo apt-get install postgresql-11 postgresql-contrib
- Clone the repo
git clone [email protected]:cfcdavidchan/HKJC.git
- Creating conda environment for the project and activate it
conda env create -f environment.yml
conda activate HKJC
- Create PostgresSQL database
sudo su - postgres
psql
\i repo_path/create_database.sql
Note: repo_path means the full path of this repo in your local computer
- Add going data into Database
python ./HKJC_database/going.py
- Get into the crawler directory
cd HKJC_crawler
- Run the crawler and store into PostgreSQL
scrapy crawl Course_crawler #crawl course data into Database
scrapy crawl Jockeys_crawler #crawl jockey data into Database
scrapy crawl Trainer_crawler #crawl trainer data into Database
scrapy crawl Horse_crawler #crawl horse data into Database
scrapy crawl Match_crawler #crawl Match data into Database