diff --git a/podman/incubating/localai/README.md b/podman/incubating/localai/README.md new file mode 100644 index 0000000..38afada --- /dev/null +++ b/podman/incubating/localai/README.md @@ -0,0 +1,135 @@ +# Local AI with Anything LLM + + + + + +## Run locally + +```bash +podman network create localai + +mkdir -p ./volumes/local-ai/local-ai +mkdir -p ./volumes/local-ai/anythingllm + +# Local AI +podman run \ + -d \ + -p 127.0.0.1:8080:8080 \ + --network localai \ + --name local-ai \ + -v ./volumes/local-ai/local-ai:/build/models \ + quay.io/go-skynet/local-ai:latest-cpu + +# Anything LLM Interface +podman run -d \ + -p 127.0.0.1:3001:3001 \ + --cap-add SYS_ADMIN \ + --network localai \ + --name anything-llm \ + -v ./volumes/local-ai/anythingllm:/app/server/storage \ + -e STORAGE_DIR="/app/server/storage" \ + docker.io/mintplexlabs/anythingllm +``` + +## Run in Production + +This installs both Local AI and Anything LLM as backend/frontend services. + +Note: + +You'll need folders for the models and anything-llm storage. + +```bash +mkdir /models +mkdir /anything-llm +chown 1000:1000 /anything-llm +``` + +```bash +podman network create localai + +# Local AI +podman run \ + -d \ + -p 127.0.0.1:8080:8080 \ + --network localai \ + --name local-ai \ + -v /models:/build/models \ + quay.io/go-skynet/local-ai:latest-cpu + +# Anything LLM Interface +podman run -d \ + -p 127.0.0.1:3001:3001 \ + --cap-add SYS_ADMIN \ + --network localai \ + --name anything-llm \ + -v /anythingllm:/app/server/storage \ + -e STORAGE_DIR="/app/server/storage" \ + docker.io/mintplexlabs/anythingllm +``` + +## Models + +### Config + +```yaml +name: llama-3.2 +parameters: + model: huggingface/Llama-3.2-3B-Instruct-f16.gguf + temperature: 0.6 +backend: llama-cpp +# Default context size +context_size: 8192 +threads: 16 +``` + +### Chat + +llama-3.2-3b-instruct:q8_0 + +### Code + +llama3.2-3b-enigma + +### Agent + +llama-3.2-3b-instruct:q8_0 + +## Nginx + +```bash +certbot-3 certonly --dns-route53 -d chatreesept.reeseapps.com +``` + +Make sure to add the following timeout configurations to your http block: + +```conf +server { + # Enable websocket connections for agent protocol. + location ~* ^/api/agent-invocation/(.*) { + proxy_pass http://0.0.0.0:3001; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + } + + listen 80; + server_name [insert FQDN here]; + location / { + # Prevent timeouts on long-running requests. + proxy_connect_timeout 605; + proxy_send_timeout 605; + proxy_read_timeout 605; + send_timeout 605; + keepalive_timeout 605; + + # Enable readable HTTP Streaming for LLM streamed responses + proxy_buffering off; + proxy_cache off; + + # Proxy your locally running service + proxy_pass http://0.0.0.0:3001; + } +} +``` \ No newline at end of file diff --git a/podman/incubating/localai/chatreesept.reeseapps.com b/podman/incubating/localai/chatreesept.reeseapps.com new file mode 100644 index 0000000..303bc7a --- /dev/null +++ b/podman/incubating/localai/chatreesept.reeseapps.com @@ -0,0 +1,32 @@ +server { + listen 127.0.0.1:8443 ssl; + server_name chatreesept.reeseapps.com; + + ssl_certificate /etc/letsencrypt/live/chatreesept.reeseapps.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/chatreesept.reeseapps.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + location ~* ^/api/agent-invocation/(.*) { + proxy_pass http://localhost:3001; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + } + + location / { + client_max_body_size 50m; + # Prevent timeouts on long-running requests. + proxy_connect_timeout 605; + proxy_send_timeout 605; + proxy_read_timeout 605; + send_timeout 605; + keepalive_timeout 605; + + # Enable readable HTTP Streaming for LLM streamed responses + proxy_buffering off; + proxy_cache off; + + proxy_pass http://localhost:3001; + } +} \ No newline at end of file