Code for a combined Llava and BridgeTower REST API server for Intel Gaudi 2
NOTE: tested on SynapseAI 1.16.2
git clone https://github.com/huggingface/optimum-habana/
cd optimum-habana/
docker run -it --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --net=host --ipc=host -v $(pwd):/workspace -v /data/$USER:/root vault.habana.ai/gaudi-docker/1.16.2/ubuntu22.04/habanalabs/pytorch-installer-2.2.2:latest
cd /workspace/optimum-habana
pip install -e .
pip install flask
python llavabridge.py --model_name_or_path llava-hf/llava-v1.6-vicuna-13b-hf --use_hpu_graphs --bf16